From 0cea64309a6ffe3617452b1c97e8ba651ac9b453 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 3 Jul 2024 21:05:24 +0000 Subject: [PATCH 001/158] move PduEvent from services to core Signed-off-by: Jason Volk --- src/api/client/membership.rs | 9 +-- src/api/server/backfill.rs | 6 +- src/api/server/event.rs | 6 +- src/api/server/event_auth.rs | 6 +- src/api/server/get_missing_events.rs | 6 +- src/api/server/invite.rs | 12 ++-- src/api/server/send_join.rs | 11 ++-- src/api/server/state.rs | 8 ++- src/core/mod.rs | 4 +- src/core/pdu/builder.rs | 16 ++++++ src/core/{pducount.rs => pdu/count.rs} | 0 src/{service/pdu.rs => core/pdu/mod.rs} | 55 ++----------------- src/service/mod.rs | 5 +- src/service/sending/mod.rs | 1 + src/service/sending/sender.rs | 73 ++++++++++++++++++------- 15 files changed, 118 insertions(+), 100 deletions(-) create mode 100644 src/core/pdu/builder.rs rename src/core/{pducount.rs => pdu/count.rs} (100%) rename src/{service/pdu.rs => core/pdu/mod.rs} (88%) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 8529a9df..bb19ee29 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -33,6 +33,7 @@ use ruma::{ OwnedUserId, RoomId, RoomVersionId, ServerName, UserId, }; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; +use service::sending::convert_to_outgoing_federation_event; use tokio::sync::RwLock; use tracing::{debug, error, info, trace, warn}; @@ -779,7 +780,7 @@ async fn join_room_by_id_helper_remote( federation::membership::create_join_event::v2::Request { room_id: room_id.to_owned(), event_id: event_id.to_owned(), - pdu: PduEvent::convert_to_outgoing_federation_event(join_event.clone()), + pdu: convert_to_outgoing_federation_event(join_event.clone()), omit_members: false, }, ) @@ -1207,7 +1208,7 @@ async fn join_room_by_id_helper_local( federation::membership::create_join_event::v2::Request { room_id: room_id.to_owned(), event_id: event_id.to_owned(), - pdu: PduEvent::convert_to_outgoing_federation_event(join_event.clone()), + pdu: convert_to_outgoing_federation_event(join_event.clone()), omit_members: false, }, ) @@ -1438,7 +1439,7 @@ pub(crate) async fn invite_helper( room_id: room_id.to_owned(), event_id: (*pdu.event_id).to_owned(), room_version: room_version_id.clone(), - event: PduEvent::convert_to_outgoing_federation_event(pdu_json.clone()), + event: convert_to_outgoing_federation_event(pdu_json.clone()), invite_room_state, via: services().rooms.state_cache.servers_route_via(room_id).ok(), }, @@ -1775,7 +1776,7 @@ async fn remote_leave_room(user_id: &UserId, room_id: &RoomId) -> Result<()> { federation::membership::create_leave_event::v2::Request { room_id: room_id.to_owned(), event_id, - pdu: PduEvent::convert_to_outgoing_federation_event(leave_event.clone()), + pdu: convert_to_outgoing_federation_event(leave_event.clone()), }, ) .await?; diff --git a/src/api/server/backfill.rs b/src/api/server/backfill.rs index e3ff82e4..b432ae20 100644 --- a/src/api/server/backfill.rs +++ b/src/api/server/backfill.rs @@ -1,9 +1,11 @@ +use conduit::{Error, Result}; use ruma::{ api::{client::error::ErrorKind, federation::backfill::get_backfill}, uint, user_id, MilliSecondsSinceUnixEpoch, }; +use service::{sending::convert_to_outgoing_federation_event, services}; -use crate::{services, Error, PduEvent, Result, Ruma}; +use crate::Ruma; /// # `GET /_matrix/federation/v1/backfill/` /// @@ -62,7 +64,7 @@ pub(crate) async fn get_backfill_route(body: Ruma) -> }) .map(|(_, pdu)| services().rooms.timeline.get_pdu_json(&pdu.event_id)) .filter_map(|r| r.ok().flatten()) - .map(PduEvent::convert_to_outgoing_federation_event) + .map(convert_to_outgoing_federation_event) .collect(); Ok(get_backfill::v1::Response { diff --git a/src/api/server/event.rs b/src/api/server/event.rs index 29f538b4..f4c9d145 100644 --- a/src/api/server/event.rs +++ b/src/api/server/event.rs @@ -1,9 +1,11 @@ +use conduit::{Error, Result}; use ruma::{ api::{client::error::ErrorKind, federation::event::get_event}, MilliSecondsSinceUnixEpoch, RoomId, }; +use service::{sending::convert_to_outgoing_federation_event, services}; -use crate::{services, Error, PduEvent, Result, Ruma}; +use crate::Ruma; /// # `GET /_matrix/federation/v1/event/{eventId}` /// @@ -48,6 +50,6 @@ pub(crate) async fn get_event_route(body: Ruma) -> Resul Ok(get_event::v1::Response { origin: services().globals.server_name().to_owned(), origin_server_ts: MilliSecondsSinceUnixEpoch::now(), - pdu: PduEvent::convert_to_outgoing_federation_event(event), + pdu: convert_to_outgoing_federation_event(event), }) } diff --git a/src/api/server/event_auth.rs b/src/api/server/event_auth.rs index 1ddf2ce3..bef5116b 100644 --- a/src/api/server/event_auth.rs +++ b/src/api/server/event_auth.rs @@ -1,11 +1,13 @@ use std::sync::Arc; +use conduit::{Error, Result}; use ruma::{ api::{client::error::ErrorKind, federation::authorization::get_event_authorization}, RoomId, }; +use service::{sending::convert_to_outgoing_federation_event, services}; -use crate::{services, Error, PduEvent, Result, Ruma}; +use crate::Ruma; /// # `GET /_matrix/federation/v1/event_auth/{roomId}/{eventId}` /// @@ -57,7 +59,7 @@ pub(crate) async fn get_event_authorization_route( Ok(get_event_authorization::v1::Response { auth_chain: auth_chain_ids .filter_map(|id| services().rooms.timeline.get_pdu_json(&id).ok()?) - .map(PduEvent::convert_to_outgoing_federation_event) + .map(convert_to_outgoing_federation_event) .collect(), }) } diff --git a/src/api/server/get_missing_events.rs b/src/api/server/get_missing_events.rs index 1c9a6a38..5ab9abf8 100644 --- a/src/api/server/get_missing_events.rs +++ b/src/api/server/get_missing_events.rs @@ -1,9 +1,11 @@ +use conduit::{Error, Result}; use ruma::{ api::{client::error::ErrorKind, federation::event::get_missing_events}, OwnedEventId, RoomId, }; +use service::{sending::convert_to_outgoing_federation_event, services}; -use crate::{services, Error, PduEvent, Result, Ruma}; +use crate::Ruma; /// # `POST /_matrix/federation/v1/get_missing_events/{roomId}` /// @@ -79,7 +81,7 @@ pub(crate) async fn get_missing_events_route( ) .map_err(|_| Error::bad_database("Invalid prev_events in event in database."))?, ); - events.push(PduEvent::convert_to_outgoing_federation_event(pdu)); + events.push(convert_to_outgoing_federation_event(pdu)); } i = i.saturating_add(1); } diff --git a/src/api/server/invite.rs b/src/api/server/invite.rs index 333ebee6..89b90058 100644 --- a/src/api/server/invite.rs +++ b/src/api/server/invite.rs @@ -1,18 +1,14 @@ use axum_client_ip::InsecureClientIp; +use conduit::{utils, warn, Error, PduEvent, Result}; use ruma::{ api::{client::error::ErrorKind, federation::membership::create_invite}, events::room::member::{MembershipState, RoomMemberEventContent}, serde::JsonObject, CanonicalJsonValue, EventId, OwnedUserId, }; -use tracing::warn; +use service::{sending::convert_to_outgoing_federation_event, server_is_ours, services}; -use crate::{ - service::server_is_ours, - services, - utils::{self}, - Error, PduEvent, Result, Ruma, -}; +use crate::Ruma; /// # `PUT /_matrix/federation/v2/invite/{roomId}/{eventId}` /// @@ -176,6 +172,6 @@ pub(crate) async fn create_invite_route( } Ok(create_invite::v2::Response { - event: PduEvent::convert_to_outgoing_federation_event(signed_event), + event: convert_to_outgoing_federation_event(signed_event), }) } diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index 82cfc45e..ff362f64 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -2,6 +2,7 @@ use std::collections::BTreeMap; +use conduit::{Error, Result}; use ruma::{ api::{client::error::ErrorKind, federation::membership::create_join_event}, events::{ @@ -11,11 +12,13 @@ use ruma::{ CanonicalJsonValue, OwnedServerName, OwnedUserId, RoomId, ServerName, }; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; -use service::user_is_local; +use service::{ + pdu::gen_event_id_canonical_json, sending::convert_to_outgoing_federation_event, services, user_is_local, +}; use tokio::sync::RwLock; use tracing::warn; -use crate::{service::pdu::gen_event_id_canonical_json, services, Error, PduEvent, Result, Ruma}; +use crate::Ruma; /// helper method for /send_join v1 and v2 async fn create_join_event( @@ -181,12 +184,12 @@ async fn create_join_event( Ok(create_join_event::v1::RoomState { auth_chain: auth_chain_ids .filter_map(|id| services().rooms.timeline.get_pdu_json(&id).ok().flatten()) - .map(PduEvent::convert_to_outgoing_federation_event) + .map(convert_to_outgoing_federation_event) .collect(), state: state_ids .iter() .filter_map(|(_, id)| services().rooms.timeline.get_pdu_json(id).ok().flatten()) - .map(PduEvent::convert_to_outgoing_federation_event) + .map(convert_to_outgoing_federation_event) .collect(), // Event field is required if the room version supports restricted join rules. event: Some( diff --git a/src/api/server/state.rs b/src/api/server/state.rs index c858f6fd..22044840 100644 --- a/src/api/server/state.rs +++ b/src/api/server/state.rs @@ -1,8 +1,10 @@ use std::sync::Arc; +use conduit::{Error, Result}; use ruma::api::{client::error::ErrorKind, federation::event::get_room_state}; +use service::{sending::convert_to_outgoing_federation_event, services}; -use crate::{services, Error, PduEvent, Result, Ruma}; +use crate::Ruma; /// # `GET /_matrix/federation/v1/state/{roomId}` /// @@ -42,7 +44,7 @@ pub(crate) async fn get_room_state_route( .await? .into_values() .map(|id| { - PduEvent::convert_to_outgoing_federation_event( + convert_to_outgoing_federation_event( services() .rooms .timeline @@ -67,7 +69,7 @@ pub(crate) async fn get_room_state_route( .timeline .get_pdu_json(&id) .ok()? - .map(PduEvent::convert_to_outgoing_federation_event) + .map(convert_to_outgoing_federation_event) }) .collect(), pdus, diff --git a/src/core/mod.rs b/src/core/mod.rs index 5ffe4cb9..ec536ee2 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -4,14 +4,14 @@ pub mod debug; pub mod error; pub mod log; pub mod mods; -pub mod pducount; +pub mod pdu; pub mod server; pub mod utils; pub mod version; pub use config::Config; pub use error::{Error, RumaResponse}; -pub use pducount::PduCount; +pub use pdu::{PduBuilder, PduCount, PduEvent}; pub use server::Server; pub use version::version; diff --git a/src/core/pdu/builder.rs b/src/core/pdu/builder.rs new file mode 100644 index 00000000..a8bad677 --- /dev/null +++ b/src/core/pdu/builder.rs @@ -0,0 +1,16 @@ +use std::{collections::BTreeMap, sync::Arc}; + +use ruma::{events::TimelineEventType, EventId}; +use serde::Deserialize; +use serde_json::value::RawValue as RawJsonValue; + +/// Build the start of a PDU in order to add it to the Database. +#[derive(Debug, Deserialize)] +pub struct PduBuilder { + #[serde(rename = "type")] + pub event_type: TimelineEventType, + pub content: Box, + pub unsigned: Option>, + pub state_key: Option, + pub redacts: Option>, +} diff --git a/src/core/pducount.rs b/src/core/pdu/count.rs similarity index 100% rename from src/core/pducount.rs rename to src/core/pdu/count.rs diff --git a/src/service/pdu.rs b/src/core/pdu/mod.rs similarity index 88% rename from src/service/pdu.rs rename to src/core/pdu/mod.rs index b5650c0a..c7d93608 100644 --- a/src/service/pdu.rs +++ b/src/core/pdu/mod.rs @@ -1,6 +1,10 @@ +mod builder; +mod count; + use std::{cmp::Ordering, collections::BTreeMap, sync::Arc}; -use conduit::{warn, Error}; +pub use builder::PduBuilder; +pub use count::PduCount; use ruma::{ canonical_json::redact_content_in_place, events::{ @@ -19,7 +23,7 @@ use serde_json::{ value::{to_raw_value, RawValue as RawJsonValue}, }; -use crate::services; +use crate::{warn, Error}; #[derive(Deserialize)] struct ExtractRedactedBecause { @@ -336,42 +340,6 @@ impl PduEvent { serde_json::from_value(json).expect("Raw::from_value always works") } - /// This does not return a full `Pdu` it is only to satisfy ruma's types. - #[tracing::instrument] - pub fn convert_to_outgoing_federation_event(mut pdu_json: CanonicalJsonObject) -> Box { - if let Some(unsigned) = pdu_json - .get_mut("unsigned") - .and_then(|val| val.as_object_mut()) - { - unsigned.remove("transaction_id"); - } - - // room v3 and above removed the "event_id" field from remote PDU format - if let Some(room_id) = pdu_json - .get("room_id") - .and_then(|val| RoomId::parse(val.as_str()?).ok()) - { - match services().rooms.state.get_room_version(&room_id) { - Ok(room_version_id) => match room_version_id { - RoomVersionId::V1 | RoomVersionId::V2 => {}, - _ => _ = pdu_json.remove("event_id"), - }, - Err(_) => _ = pdu_json.remove("event_id"), - } - } else { - pdu_json.remove("event_id"); - } - - // TODO: another option would be to convert it to a canonical string to validate - // size and return a Result> - // serde_json::from_str::>( - // ruma::serde::to_canonical_json_string(pdu_json).expect("CanonicalJson is - // valid serde_json::Value"), ) - // .expect("Raw::from_value always works") - - to_raw_value(&pdu_json).expect("CanonicalJson is valid serde_json::Value") - } - pub fn from_id_val(event_id: &EventId, mut json: CanonicalJsonObject) -> Result { json.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.as_str().to_owned())); @@ -438,14 +406,3 @@ pub fn gen_event_id_canonical_json( Ok((event_id, value)) } - -/// Build the start of a PDU in order to add it to the Database. -#[derive(Debug, Deserialize)] -pub struct PduBuilder { - #[serde(rename = "type")] - pub event_type: TimelineEventType, - pub content: Box, - pub unsigned: Option>, - pub state_key: Option, - pub redacts: Option>, -} diff --git a/src/service/mod.rs b/src/service/mod.rs index c80c9862..641dd36a 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -1,4 +1,3 @@ -pub mod pdu; pub mod services; pub mod account_data; @@ -20,12 +19,12 @@ extern crate conduit_database as database; use std::sync::{Arc, RwLock}; -pub(crate) use conduit::{config, debug_error, debug_info, debug_warn, utils, Config, Error, PduCount, Result, Server}; +pub(crate) use conduit::{config, debug_error, debug_info, debug_warn, utils, Config, Error, Result, Server}; +pub use conduit::{pdu, PduBuilder, PduCount, PduEvent}; use database::Database; pub use crate::{ globals::{server_is_ours, user_is_local}, - pdu::PduEvent, services::Services, }; diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index 9bda6cad..c57ca14d 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -14,6 +14,7 @@ use ruma::{ api::{appservice::Registration, OutgoingRequest}, OwnedServerName, OwnedUserId, RoomId, ServerName, UserId, }; +pub use sender::convert_to_outgoing_federation_event; use tokio::{sync::Mutex, task::JoinHandle}; use tracing::{error, warn}; diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index aa2865ec..61cb9f6d 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -18,12 +18,14 @@ use ruma::{ }, device_id, events::{push_rules::PushRulesEvent, receipt::ReceiptType, AnySyncEphemeralRoomEvent, GlobalAccountDataEventType}, - push, uint, MilliSecondsSinceUnixEpoch, OwnedServerName, OwnedUserId, RoomId, ServerName, UInt, + push, uint, CanonicalJsonObject, MilliSecondsSinceUnixEpoch, OwnedServerName, OwnedUserId, RoomId, RoomVersionId, + ServerName, UInt, }; +use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; use tracing::{debug, error, warn}; use super::{appservice, send, Destination, Msg, SendingEvent, Service}; -use crate::{presence::Presence, services, user_is_local, utils::calculate_hash, Error, PduEvent, Result}; +use crate::{presence::Presence, services, user_is_local, utils::calculate_hash, Error, Result}; #[derive(Debug)] enum TransactionStatus { @@ -548,24 +550,21 @@ async fn send_events_dest_normal( for event in &events { match event { - SendingEvent::Pdu(pdu_id) => { + SendingEvent::Pdu(pdu_id) => pdu_jsons.push(convert_to_outgoing_federation_event( // TODO: check room version and remove event_id if needed - let raw = PduEvent::convert_to_outgoing_federation_event( - services() - .rooms - .timeline - .get_pdu_json_from_id(pdu_id) - .map_err(|e| (dest.clone(), e))? - .ok_or_else(|| { - error!(?dest, ?server, ?pdu_id, "event not found"); - ( - dest.clone(), - Error::bad_database("[Normal] Event in servernameevent_data not found in db."), - ) - })?, - ); - pdu_jsons.push(raw); - }, + services() + .rooms + .timeline + .get_pdu_json_from_id(pdu_id) + .map_err(|e| (dest.clone(), e))? + .ok_or_else(|| { + error!(?dest, ?server, ?pdu_id, "event not found"); + ( + dest.clone(), + Error::bad_database("[Normal] Event in servernameevent_data not found in db."), + ) + })?, + )), SendingEvent::Edu(edu) => { if let Ok(raw) = serde_json::from_slice(edu) { edu_jsons.push(raw); @@ -611,3 +610,39 @@ async fn send_events_dest_normal( }) .map_err(|e| (dest.clone(), e)) } + +/// This does not return a full `Pdu` it is only to satisfy ruma's types. +#[tracing::instrument] +pub fn convert_to_outgoing_federation_event(mut pdu_json: CanonicalJsonObject) -> Box { + if let Some(unsigned) = pdu_json + .get_mut("unsigned") + .and_then(|val| val.as_object_mut()) + { + unsigned.remove("transaction_id"); + } + + // room v3 and above removed the "event_id" field from remote PDU format + if let Some(room_id) = pdu_json + .get("room_id") + .and_then(|val| RoomId::parse(val.as_str()?).ok()) + { + match services().rooms.state.get_room_version(&room_id) { + Ok(room_version_id) => match room_version_id { + RoomVersionId::V1 | RoomVersionId::V2 => {}, + _ => _ = pdu_json.remove("event_id"), + }, + Err(_) => _ = pdu_json.remove("event_id"), + } + } else { + pdu_json.remove("event_id"); + } + + // TODO: another option would be to convert it to a canonical string to validate + // size and return a Result> + // serde_json::from_str::>( + // ruma::serde::to_canonical_json_string(pdu_json).expect("CanonicalJson is + // valid serde_json::Value"), ) + // .expect("Raw::from_value always works") + + to_raw_value(&pdu_json).expect("CanonicalJson is valid serde_json::Value") +} From f104ced55dc19559fc380846ee57684c6f670428 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 3 Jul 2024 20:19:20 +0000 Subject: [PATCH 002/158] remove unnecessary wrapper Signed-off-by: Jason Volk --- src/service/globals/mod.rs | 3 --- src/service/sending/resolve.rs | 6 ++++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 7e1ccb5e..27be1516 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -14,7 +14,6 @@ use std::{ use conduit::{error, trace, utils::MutexMap, Config, Result, Server}; use data::Data; use database::Database; -use hickory_resolver::TokioAsyncResolver; use ipaddress::IPAddress; use regex::RegexSet; use ruma::{ @@ -194,8 +193,6 @@ impl Service { pub fn query_trusted_key_servers_first(&self) -> bool { self.config.query_trusted_key_servers_first } - pub fn dns_resolver(&self) -> &TokioAsyncResolver { &self.resolver.resolver } - pub fn jwt_decoding_key(&self) -> Option<&jsonwebtoken::DecodingKey> { self.jwt_decoding_key.as_ref() } pub fn turn_password(&self) -> &String { &self.config.turn_password } diff --git a/src/service/sending/resolve.rs b/src/service/sending/resolve.rs index 294ac09f..51e7a92f 100644 --- a/src/service/sending/resolve.rs +++ b/src/service/sending/resolve.rs @@ -261,7 +261,8 @@ async fn conditional_query_and_cache_override(overname: &str, hostname: &str, po async fn query_and_cache_override(overname: &'_ str, hostname: &'_ str, port: u16) -> Result<()> { match services() .globals - .dns_resolver() + .resolver + .resolver .lookup_ip(hostname.to_owned()) .await { @@ -299,7 +300,8 @@ async fn query_srv_record(hostname: &'_ str) -> Result> { let hostname = hostname.trim_end_matches('.'); services() .globals - .dns_resolver() + .resolver + .resolver .srv_lookup(hostname.to_owned()) .await } From 25c004f08cc13ef830d380056e326078a4fe6708 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 3 Jul 2024 23:11:05 +0000 Subject: [PATCH 003/158] remove dead cache Signed-off-by: Jason Volk --- src/service/rooms/timeline/mod.rs | 11 ++++------- src/service/services.rs | 10 ++-------- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index b9c0d7ba..dfcfedea 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -1,7 +1,7 @@ mod data; use std::{ - collections::{BTreeMap, HashMap, HashSet}, + collections::{BTreeMap, HashSet}, sync::Arc, }; @@ -27,12 +27,12 @@ use ruma::{ push::{Action, Ruleset, Tweak}, serde::Base64, state_res::{self, Event, RoomVersion}, - uint, user_id, CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedEventId, OwnedRoomId, OwnedServerName, - RoomId, RoomVersionId, ServerName, UserId, + uint, user_id, CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedEventId, OwnedServerName, RoomId, + RoomVersionId, ServerName, UserId, }; use serde::Deserialize; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; -use tokio::sync::{Mutex, RwLock}; +use tokio::sync::RwLock; use crate::{ admin, @@ -66,15 +66,12 @@ struct ExtractBody { pub struct Service { db: Data, - - pub lasttimelinecount_cache: Mutex>, } impl Service { pub fn build(_server: &Arc, db: &Arc) -> Result { Ok(Self { db: Data::new(db), - lasttimelinecount_cache: Mutex::new(HashMap::new()), }) } diff --git a/src/service/services.rs b/src/service/services.rs index 70e6a6d5..81aba231 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -95,10 +95,7 @@ impl Services { let lasttimelinecount_cache = self .rooms .timeline - .lasttimelinecount_cache - .lock() - .await - .len(); + .get_lasttimelinecount_cache_usage().0; let roomid_spacehierarchy_cache = self .rooms .spaces @@ -177,10 +174,7 @@ bad_signature_ratelimiter: {bad_signature_ratelimiter} if amount > 4 { self.rooms .timeline - .lasttimelinecount_cache - .lock() - .await - .clear(); + .clear_lasttimelinecount_cache(); } if amount > 5 { self.rooms From 4f5c6de8534e35455d44c55211741975aefb0e2a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 3 Jul 2024 08:44:59 +0000 Subject: [PATCH 004/158] start rand utils suite Signed-off-by: Jason Volk --- src/core/utils/mod.rs | 6 ++++-- src/core/utils/rand.rs | 24 ++++++++++++++++++++++++ src/core/utils/string.rs | 10 ---------- 3 files changed, 28 insertions(+), 12 deletions(-) create mode 100644 src/core/utils/rand.rs diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 401775f7..75f55c9a 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -6,6 +6,7 @@ pub mod hash; pub mod html; pub mod json; pub mod mutex_map; +pub mod rand; pub mod string; pub mod sys; mod tests; @@ -19,7 +20,8 @@ pub use hash::calculate_hash; pub use html::Escape as HtmlEscape; pub use json::{deserialize_from_str, to_canonical_object}; pub use mutex_map::MutexMap; -pub use string::{random_string, str_from_bytes, string_from_bytes}; +pub use rand::string as random_string; +pub use string::{str_from_bytes, string_from_bytes}; pub use sys::available_parallelism; pub use time::millis_since_unix_epoch; @@ -41,7 +43,7 @@ pub fn unwrap_infallible(result: Result) -> T { #[must_use] pub fn generate_keypair() -> Vec { - let mut value = random_string(8).as_bytes().to_vec(); + let mut value = rand::string(8).as_bytes().to_vec(); value.push(0xFF); value.extend_from_slice( &ruma::signatures::Ed25519KeyPair::generate().expect("Ed25519KeyPair generation always works (?)"), diff --git a/src/core/utils/rand.rs b/src/core/utils/rand.rs new file mode 100644 index 00000000..1ded8a6d --- /dev/null +++ b/src/core/utils/rand.rs @@ -0,0 +1,24 @@ +use std::{ + ops::Range, + time::{Duration, SystemTime}, +}; + +use rand::{thread_rng, Rng}; + +pub fn string(length: usize) -> String { + thread_rng() + .sample_iter(&rand::distributions::Alphanumeric) + .take(length) + .map(char::from) + .collect() +} + +#[inline] +#[must_use] +pub fn timepoint_secs(range: Range) -> SystemTime { SystemTime::now() + secs(range) } + +#[must_use] +pub fn secs(range: Range) -> Duration { + let mut rng = thread_rng(); + Duration::from_secs(rng.gen_range(range)) +} diff --git a/src/core/utils/string.rs b/src/core/utils/string.rs index 7f6f6531..84a928a0 100644 --- a/src/core/utils/string.rs +++ b/src/core/utils/string.rs @@ -1,15 +1,5 @@ -use rand::prelude::*; - use crate::Result; -pub fn random_string(length: usize) -> String { - thread_rng() - .sample_iter(&rand::distributions::Alphanumeric) - .take(length) - .map(char::from) - .collect() -} - /// Parses the bytes into a string. #[inline] pub fn string_from_bytes(bytes: &[u8]) -> Result { From 177c9e8bfa87aab3d483990c333c4fe4e2646dd2 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 3 Jul 2024 23:12:43 +0000 Subject: [PATCH 005/158] add split_once_infallible string util Signed-off-by: Jason Volk --- src/core/utils/string.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/core/utils/string.rs b/src/core/utils/string.rs index 84a928a0..78765154 100644 --- a/src/core/utils/string.rs +++ b/src/core/utils/string.rs @@ -1,5 +1,13 @@ use crate::Result; +pub const EMPTY: &str = ""; + +#[inline] +#[must_use] +pub fn split_once_infallible<'a>(input: &'a str, delim: &'_ str) -> (&'a str, &'a str) { + input.split_once(delim).unwrap_or((input, EMPTY)) +} + /// Parses the bytes into a string. #[inline] pub fn string_from_bytes(bytes: &[u8]) -> Result { From e125af620eba5ed15c1278bf9b6bad1f05c3086a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 4 Jul 2024 03:26:19 +0000 Subject: [PATCH 006/158] impl crate::Service for Service Signed-off-by: Jason Volk --- src/admin/federation/commands.rs | 10 +- src/admin/server/commands.rs | 16 +- src/admin/server/mod.rs | 20 +- src/api/client/keys.rs | 4 +- src/api/client/membership.rs | 4 +- src/core/alloc/je.rs | 2 +- src/service/account_data/mod.rs | 17 +- src/service/admin/mod.rs | 23 +- src/service/appservice/mod.rs | 17 +- src/service/globals/data.rs | 35 --- src/service/globals/mod.rs | 71 +++++- src/service/globals/resolver.rs | 21 +- src/service/key_backups/mod.rs | 17 +- src/service/media/mod.rs | 17 +- src/service/mod.rs | 6 +- src/service/presence/mod.rs | 29 ++- src/service/pusher/mod.rs | 17 +- src/service/rooms/alias/mod.rs | 17 +- src/service/rooms/auth_chain/mod.rs | 17 +- src/service/rooms/directory/mod.rs | 16 +- src/service/rooms/event_handler/mod.rs | 29 ++- src/service/rooms/lazy_loading/mod.rs | 33 ++- src/service/rooms/metadata/mod.rs | 17 +- src/service/rooms/mod.rs | 42 ++-- src/service/rooms/outlier/mod.rs | 17 +- src/service/rooms/pdu_metadata/mod.rs | 17 +- src/service/rooms/read_receipt/mod.rs | 17 +- src/service/rooms/search/mod.rs | 17 +- src/service/rooms/short/mod.rs | 17 +- src/service/rooms/spaces/mod.rs | 17 +- src/service/rooms/state/mod.rs | 17 +- src/service/rooms/state_accessor/mod.rs | 35 ++- src/service/rooms/state_cache/mod.rs | 17 +- src/service/rooms/state_compressor/mod.rs | 29 ++- src/service/rooms/threads/mod.rs | 17 +- src/service/rooms/timeline/mod.rs | 51 ++-- src/service/rooms/typing/mod.rs | 15 +- src/service/rooms/user/mod.rs | 17 +- src/service/sending/mod.rs | 27 +- src/service/service.rs | 53 ++++ src/service/services.rs | 293 +++++++--------------- src/service/transaction_ids/mod.rs | 17 +- src/service/uiaa/mod.rs | 17 +- src/service/users/mod.rs | 17 +- 44 files changed, 673 insertions(+), 548 deletions(-) create mode 100644 src/service/service.rs diff --git a/src/admin/federation/commands.rs b/src/admin/federation/commands.rs index 293efdc5..24f4bc23 100644 --- a/src/admin/federation/commands.rs +++ b/src/admin/federation/commands.rs @@ -15,14 +15,18 @@ pub(super) async fn enable_room(_body: Vec<&str>, room_id: Box) -> Resul } pub(super) async fn incoming_federation(_body: Vec<&str>) -> Result { - let map = services().globals.roomid_federationhandletime.read().await; + let map = services() + .globals + .roomid_federationhandletime + .read() + .expect("locked"); let mut msg = format!("Handling {} incoming pdus:\n", map.len()); for (r, (e, i)) in map.iter() { let elapsed = i.elapsed(); - writeln!(msg, "{} {}: {}m{}s", r, e, elapsed.as_secs() / 60, elapsed.as_secs() % 60,) - .expect("should be able to write to string buffer"); + writeln!(msg, "{} {}: {}m{}s", r, e, elapsed.as_secs() / 60, elapsed.as_secs() % 60)?; } + Ok(RoomMessageEventContent::text_plain(&msg)) } diff --git a/src/admin/server/commands.rs b/src/admin/server/commands.rs index 77b20377..2ab71033 100644 --- a/src/admin/server/commands.rs +++ b/src/admin/server/commands.rs @@ -27,12 +27,12 @@ pub(super) async fn show_config(_body: Vec<&str>) -> Result) -> Result { - let response0 = services().memory_usage().await; - let response1 = services().globals.db.memory_usage(); + let response0 = services().memory_usage().await?; + let response1 = services().db.db.memory_usage()?; let response2 = conduit::alloc::memory_usage(); Ok(RoomMessageEventContent::text_plain(format!( - "Services:\n{response0}\n\nDatabase:\n{response1}\n{}", + "Services:\n{response0}\nDatabase:\n{response1}\n{}", if !response2.is_empty() { format!("Allocator:\n {response2}") } else { @@ -41,14 +41,8 @@ pub(super) async fn memory_usage(_body: Vec<&str>) -> Result, amount: u32) -> Result { - services().globals.db.clear_caches(amount); - - Ok(RoomMessageEventContent::text_plain("Done.")) -} - -pub(super) async fn clear_service_caches(_body: Vec<&str>, amount: u32) -> Result { - services().clear_caches(amount).await; +pub(super) async fn clear_caches(_body: Vec<&str>) -> Result { + services().clear_cache().await; Ok(RoomMessageEventContent::text_plain("Done.")) } diff --git a/src/admin/server/mod.rs b/src/admin/server/mod.rs index b4222251..41b10180 100644 --- a/src/admin/server/mod.rs +++ b/src/admin/server/mod.rs @@ -18,17 +18,8 @@ pub(super) enum ServerCommand { /// - Print database memory usage statistics MemoryUsage, - /// - Clears all of Conduit's database caches with index smaller than the - /// amount - ClearDatabaseCaches { - amount: u32, - }, - - /// - Clears all of Conduit's service caches with index smaller than the - /// amount - ClearServiceCaches { - amount: u32, - }, + /// - Clears all of Conduwuit's caches + ClearCaches, /// - Performs an online backup of the database (only available for RocksDB /// at the moment) @@ -65,12 +56,7 @@ pub(super) async fn process(command: ServerCommand, body: Vec<&str>) -> Result uptime(body).await?, ServerCommand::ShowConfig => show_config(body).await?, ServerCommand::MemoryUsage => memory_usage(body).await?, - ServerCommand::ClearDatabaseCaches { - amount, - } => clear_database_caches(body, amount).await?, - ServerCommand::ClearServiceCaches { - amount, - } => clear_service_caches(body, amount).await?, + ServerCommand::ClearCaches => clear_caches(body).await?, ServerCommand::ListBackups => list_backups(body).await?, ServerCommand::BackupDatabase => backup_database(body).await?, ServerCommand::ListDatabaseFiles => list_database_files(body).await?, diff --git a/src/api/client/keys.rs b/src/api/client/keys.rs index 0cfb0138..6f089875 100644 --- a/src/api/client/keys.rs +++ b/src/api/client/keys.rs @@ -334,7 +334,7 @@ pub(crate) async fn get_keys_helper bool + Send>( .globals .bad_query_ratelimiter .write() - .await + .expect("locked") .entry(id) { hash_map::Entry::Vacant(e) => { @@ -353,7 +353,7 @@ pub(crate) async fn get_keys_helper bool + Send>( .globals .bad_query_ratelimiter .read() - .await + .expect("locked") .get(server) { // Exponential backoff diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index bb19ee29..07a585fd 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -1343,7 +1343,7 @@ pub async fn validate_and_add_event_id( .globals .bad_event_ratelimiter .write() - .await + .expect("locked") .entry(id) { Entry::Vacant(e) => { @@ -1359,7 +1359,7 @@ pub async fn validate_and_add_event_id( .globals .bad_event_ratelimiter .read() - .await + .expect("locked") .get(&event_id) { // Exponential backoff diff --git a/src/core/alloc/je.rs b/src/core/alloc/je.rs index e0628ef3..966cbde3 100644 --- a/src/core/alloc/je.rs +++ b/src/core/alloc/je.rs @@ -19,7 +19,7 @@ pub fn memory_usage() -> String { let resident = stats::resident::read().unwrap_or_default() as f64 / 1024.0 / 1024.0; let retained = stats::retained::read().unwrap_or_default() as f64 / 1024.0 / 1024.0; format!( - " allocated: {allocated:.2} MiB\n active: {active:.2} MiB\n mapped: {mapped:.2} MiB\n metadata: {metadata:.2} \ + "allocated: {allocated:.2} MiB\n active: {active:.2} MiB\n mapped: {mapped:.2} MiB\n metadata: {metadata:.2} \ MiB\n resident: {resident:.2} MiB\n retained: {retained:.2} MiB\n " ) } diff --git a/src/service/account_data/mod.rs b/src/service/account_data/mod.rs index 62560500..66474370 100644 --- a/src/service/account_data/mod.rs +++ b/src/service/account_data/mod.rs @@ -2,9 +2,8 @@ mod data; use std::{collections::HashMap, sync::Arc}; -use conduit::{Result, Server}; +use conduit::Result; use data::Data; -use database::Database; use ruma::{ events::{AnyEphemeralRoomEvent, RoomAccountDataEventType}, serde::Raw, @@ -15,13 +14,17 @@ pub struct Service { db: Data, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { /// Places one event in the account data of the user and removes the /// previous entry. #[allow(clippy::needless_pass_by_value)] diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index b5cb8af4..ed2ac725 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -4,9 +4,9 @@ mod grant; use std::{future::Future, pin::Pin, sync::Arc}; -use conduit::{error, utils::mutex_map, Error, Result, Server}; +use async_trait::async_trait; +use conduit::{error, utils::mutex_map, Error, Result}; pub use create::create_admin_room; -use database::Database; pub use grant::make_user_admin; use loole::{Receiver, Sender}; use ruma::{ @@ -43,8 +43,9 @@ pub struct Command { pub reply_id: Option, } -impl Service { - pub fn build(_server: &Arc, _db: &Arc) -> Result> { +#[async_trait] +impl crate::Service for Service { + fn build(_args: crate::Args<'_>) -> Result> { let (sender, receiver) = loole::bounded(COMMAND_QUEUE_LIMIT); Ok(Arc::new(Self { sender, @@ -56,8 +57,8 @@ impl Service { })) } - pub async fn start_handler(self: &Arc) { - let self_ = Arc::clone(self); + async fn start(self: Arc) -> Result<()> { + let self_ = Arc::clone(&self); let handle = services().server.runtime().spawn(async move { self_ .handler() @@ -66,9 +67,11 @@ impl Service { }); _ = self.handler_join.lock().await.insert(handle); + + Ok(()) } - pub fn interrupt(&self) { + fn interrupt(&self) { #[cfg(feature = "console")] self.console.interrupt(); @@ -77,7 +80,7 @@ impl Service { } } - pub async fn close(&self) { + async fn stop(&self) { self.interrupt(); #[cfg(feature = "console")] @@ -90,6 +93,10 @@ impl Service { } } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { pub async fn send_text(&self, body: &str) { self.send_message(RoomMessageEventContent::text_markdown(body)) .await; diff --git a/src/service/appservice/mod.rs b/src/service/appservice/mod.rs index 916e7df3..9bb00df1 100644 --- a/src/service/appservice/mod.rs +++ b/src/service/appservice/mod.rs @@ -2,9 +2,8 @@ mod data; use std::{collections::BTreeMap, sync::Arc}; -use conduit::{Result, Server}; +use conduit::Result; use data::Data; -use database::Database; use futures_util::Future; use regex::RegexSet; use ruma::{ @@ -119,10 +118,10 @@ pub struct Service { registration_info: RwLock>, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { let mut registration_info = BTreeMap::new(); - let db = Data::new(db); + let db = Data::new(args.db); // Inserting registrations into cache for appservice in iter_ids(&db)? { registration_info.insert( @@ -134,12 +133,16 @@ impl Service { ); } - Ok(Self { + Ok(Arc::new(Self { db, registration_info: RwLock::new(registration_info), - }) + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { pub fn all(&self) -> Result> { iter_ids(&self.db) } /// Registers an appservice and returns the ID to the caller diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index 022762f4..43614854 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -208,41 +208,6 @@ impl Data { pub fn cleanup(&self) -> Result<()> { self.db.db.cleanup() } - pub fn memory_usage(&self) -> String { - let (auth_chain_cache, max_auth_chain_cache) = services().rooms.auth_chain.get_cache_usage(); - let (appservice_in_room_cache, max_appservice_in_room_cache) = services() - .rooms - .state_cache - .get_appservice_in_room_cache_usage(); - let (lasttimelinecount_cache, max_lasttimelinecount_cache) = services() - .rooms - .timeline - .get_lasttimelinecount_cache_usage(); - - format!( - "auth_chain_cache: {auth_chain_cache} / {max_auth_chain_cache}\nappservice_in_room_cache: \ - {appservice_in_room_cache} / {max_appservice_in_room_cache}\nlasttimelinecount_cache: \ - {lasttimelinecount_cache} / {max_lasttimelinecount_cache}\n\n{}", - self.db.db.memory_usage().unwrap_or_default() - ) - } - - #[allow(clippy::unused_self)] - pub fn clear_caches(&self, amount: u32) { - if amount > 1 { - services().rooms.auth_chain.clear_cache(); - } - if amount > 2 { - services() - .rooms - .state_cache - .clear_appservice_in_room_cache(); - } - if amount > 3 { - services().rooms.timeline.clear_lasttimelinecount_cache(); - } - } - pub fn load_keypair(&self) -> Result { let keypair_bytes = self.global.get(b"keypair")?.map_or_else( || { diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 27be1516..2a6fbc2f 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -7,13 +7,13 @@ pub(super) mod updates; use std::{ collections::{BTreeMap, HashMap}, - sync::Arc, + fmt::Write, + sync::{Arc, RwLock}, time::Instant, }; -use conduit::{error, trace, utils::MutexMap, Config, Result, Server}; +use conduit::{error, trace, utils::MutexMap, Config, Result}; use data::Data; -use database::Database; use ipaddress::IPAddress; use regex::RegexSet; use ruma::{ @@ -25,10 +25,7 @@ use ruma::{ DeviceId, OwnedEventId, OwnedRoomAliasId, OwnedRoomId, OwnedServerName, OwnedServerSigningKeyId, OwnedUserId, RoomAliasId, RoomVersionId, ServerName, UserId, }; -use tokio::{ - sync::{Mutex, RwLock}, - task::JoinHandle, -}; +use tokio::{sync::Mutex, task::JoinHandle}; use url::Url; use crate::services; @@ -59,10 +56,10 @@ pub struct Service { pub admin_alias: OwnedRoomAliasId, } -impl Service { - pub fn build(server: &Arc, db: &Arc) -> Result { - let config = &server.config; - let db = Data::new(db); +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + let config = &args.server.config; + let db = Data::new(args.db); let keypair = db.load_keypair(); let keypair = match keypair { @@ -133,9 +130,59 @@ impl Service { s.config.default_room_version = crate::config::default_default_room_version(); }; - Ok(s) + Ok(Arc::new(s)) } + fn memory_usage(&self, out: &mut dyn Write) -> Result<()> { + self.resolver.memory_usage(out)?; + + let bad_event_ratelimiter = self + .bad_event_ratelimiter + .read() + .expect("locked for reading") + .len(); + writeln!(out, "bad_event_ratelimiter: {bad_event_ratelimiter}")?; + + let bad_query_ratelimiter = self + .bad_query_ratelimiter + .read() + .expect("locked for reading") + .len(); + writeln!(out, "bad_query_ratelimiter: {bad_query_ratelimiter}")?; + + let bad_signature_ratelimiter = self + .bad_signature_ratelimiter + .read() + .expect("locked for reading") + .len(); + writeln!(out, "bad_signature_ratelimiter: {bad_signature_ratelimiter}")?; + + Ok(()) + } + + fn clear_cache(&self) { + self.resolver.clear_cache(); + + self.bad_event_ratelimiter + .write() + .expect("locked for writing") + .clear(); + + self.bad_query_ratelimiter + .write() + .expect("locked for writing") + .clear(); + + self.bad_signature_ratelimiter + .write() + .expect("locked for writing") + .clear(); + } + + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { /// Returns this server's keypair. pub fn keypair(&self) -> &ruma::signatures::Ed25519KeyPair { &self.keypair } diff --git a/src/service/globals/resolver.rs b/src/service/globals/resolver.rs index 86fa6700..e5199162 100644 --- a/src/service/globals/resolver.rs +++ b/src/service/globals/resolver.rs @@ -1,12 +1,13 @@ use std::{ collections::HashMap, + fmt::Write, future, iter, net::{IpAddr, SocketAddr}, sync::{Arc, RwLock}, time::Duration, }; -use conduit::{error, Config, Error}; +use conduit::{error, Config, Error, Result}; use hickory_resolver::TokioAsyncResolver; use reqwest::dns::{Addrs, Name, Resolve, Resolving}; use ruma::OwnedServerName; @@ -30,7 +31,7 @@ pub struct Hooked { impl Resolver { #[allow(clippy::as_conversions, clippy::cast_sign_loss, clippy::cast_possible_truncation)] - pub fn new(config: &Config) -> Self { + pub(super) fn new(config: &Config) -> Self { let (sys_conf, mut opts) = hickory_resolver::system_conf::read_system_conf() .map_err(|e| { error!("Failed to set up hickory dns resolver with system config: {e}"); @@ -92,6 +93,22 @@ impl Resolver { }), } } + + pub(super) fn memory_usage(&self, out: &mut dyn Write) -> Result<()> { + let resolver_overrides_cache = self.overrides.read().expect("locked for reading").len(); + writeln!(out, "resolver_overrides_cache: {resolver_overrides_cache}")?; + + let resolver_destinations_cache = self.destinations.read().expect("locked for reading").len(); + writeln!(out, "resolver_destinations_cache: {resolver_destinations_cache}")?; + + Ok(()) + } + + pub(super) fn clear_cache(&self) { + self.overrides.write().expect("write locked").clear(); + self.destinations.write().expect("write locked").clear(); + self.resolver.clear_cache(); + } } impl Resolve for Resolver { diff --git a/src/service/key_backups/mod.rs b/src/service/key_backups/mod.rs index 650aa6b6..d83d4497 100644 --- a/src/service/key_backups/mod.rs +++ b/src/service/key_backups/mod.rs @@ -2,9 +2,8 @@ mod data; use std::{collections::BTreeMap, sync::Arc}; -use conduit::{Result, Server}; +use conduit::Result; use data::Data; -use database::Database; use ruma::{ api::client::backup::{BackupAlgorithm, KeyBackupData, RoomKeyBackup}, serde::Raw, @@ -15,13 +14,17 @@ pub struct Service { db: Data, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { pub fn create_backup(&self, user_id: &UserId, backup_metadata: &Raw) -> Result { self.db.create_backup(user_id, backup_metadata) } diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index 0323bfc5..caa75d4e 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -6,7 +6,6 @@ use std::{collections::HashMap, io::Cursor, path::PathBuf, sync::Arc, time::Syst use base64::{engine::general_purpose, Engine as _}; use conduit::{debug, debug_error, error, utils, Error, Result, Server}; use data::Data; -use database::Database; use image::imageops::FilterType; use ruma::{OwnedMxcUri, OwnedUserId}; use serde::Serialize; @@ -48,15 +47,19 @@ pub struct Service { pub url_preview_mutex: RwLock>>>, } -impl Service { - pub fn build(server: &Arc, db: &Arc) -> Result { - Ok(Self { - server: server.clone(), - db: Data::new(db), +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + server: args.server.clone(), + db: Data::new(args.db), url_preview_mutex: RwLock::new(HashMap::new()), - }) + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { /// Uploads a file. pub async fn create( &self, sender_user: Option, mxc: &str, content_disposition: Option<&str>, diff --git a/src/service/mod.rs b/src/service/mod.rs index 641dd36a..9c83b25c 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -1,3 +1,6 @@ +#![allow(refining_impl_trait)] + +mod service; pub mod services; pub mod account_data; @@ -22,6 +25,7 @@ use std::sync::{Arc, RwLock}; pub(crate) use conduit::{config, debug_error, debug_info, debug_warn, utils, Config, Error, Result, Server}; pub use conduit::{pdu, PduBuilder, PduCount, PduEvent}; use database::Database; +pub(crate) use service::{Args, Service}; pub use crate::{ globals::{server_is_ours, user_is_local}, @@ -36,7 +40,7 @@ static SERVICES: RwLock> = RwLock::new(None); #[allow(clippy::let_underscore_must_use)] pub async fn init(server: &Arc) -> Result<()> { let d = Arc::new(Database::open(server).await?); - let s = Box::new(Services::build(server.clone(), d.clone()).await?); + let s = Box::new(Services::build(server.clone(), d)?); _ = SERVICES.write().expect("write locked").insert(Box::leak(s)); Ok(()) diff --git a/src/service/presence/mod.rs b/src/service/presence/mod.rs index 5065efdd..6f05d402 100644 --- a/src/service/presence/mod.rs +++ b/src/service/presence/mod.rs @@ -2,9 +2,9 @@ mod data; use std::{sync::Arc, time::Duration}; -use conduit::{debug, error, utils, Error, Result, Server}; +use async_trait::async_trait; +use conduit::{debug, error, utils, Error, Result}; use data::Data; -use database::Database; use futures_util::{stream::FuturesUnordered, StreamExt}; use ruma::{ events::presence::{PresenceEvent, PresenceEventContent}, @@ -81,12 +81,13 @@ pub struct Service { timeout_remote_users: bool, } -impl Service { - pub fn build(server: &Arc, db: &Arc) -> Result> { - let config = &server.config; +#[async_trait] +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + let config = &args.server.config; let (timer_sender, timer_receiver) = loole::unbounded(); Ok(Arc::new(Self { - db: Data::new(db), + db: Data::new(args.db), timer_sender, timer_receiver: Mutex::new(timer_receiver), handler_join: Mutex::new(None), @@ -94,8 +95,10 @@ impl Service { })) } - pub async fn start_handler(self: &Arc) { - let self_ = Arc::clone(self); + async fn start(self: Arc) -> Result<()> { + //TODO: if self.globals.config.allow_local_presence { return; } + + let self_ = Arc::clone(&self); let handle = services().server.runtime().spawn(async move { self_ .handler() @@ -104,9 +107,11 @@ impl Service { }); _ = self.handler_join.lock().await.insert(handle); + + Ok(()) } - pub async fn close(&self) { + async fn stop(&self) { self.interrupt(); if let Some(handler_join) = self.handler_join.lock().await.take() { if let Err(e) = handler_join.await { @@ -115,12 +120,16 @@ impl Service { } } - pub fn interrupt(&self) { + fn interrupt(&self) { if !self.timer_sender.is_closed() { self.timer_sender.close(); } } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { /// Returns the latest presence event for the given user. pub fn get_presence(&self, user_id: &UserId) -> Result> { if let Some((_, presence)) = self.db.get_presence(user_id)? { diff --git a/src/service/pusher/mod.rs b/src/service/pusher/mod.rs index 28280273..27b49c28 100644 --- a/src/service/pusher/mod.rs +++ b/src/service/pusher/mod.rs @@ -3,9 +3,8 @@ mod data; use std::{fmt::Debug, mem, sync::Arc}; use bytes::BytesMut; -use conduit::{debug_info, info, trace, warn, Error, Result, Server}; +use conduit::{debug_info, info, trace, warn, Error, Result}; use data::Data; -use database::Database; use ipaddress::IPAddress; use ruma::{ api::{ @@ -30,13 +29,17 @@ pub struct Service { db: Data, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { pub fn set_pusher(&self, sender: &UserId, pusher: &set_pusher::v3::PusherAction) -> Result<()> { self.db.set_pusher(sender, pusher) } diff --git a/src/service/rooms/alias/mod.rs b/src/service/rooms/alias/mod.rs index 9f66cf88..d375561e 100644 --- a/src/service/rooms/alias/mod.rs +++ b/src/service/rooms/alias/mod.rs @@ -3,9 +3,8 @@ mod remote; use std::sync::Arc; -use conduit::{Error, Result, Server}; +use conduit::{Error, Result}; use data::Data; -use database::Database; use ruma::{ api::{appservice, client::error::ErrorKind}, events::{ @@ -21,13 +20,17 @@ pub struct Service { db: Data, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { #[tracing::instrument(skip(self))] pub fn set_alias(&self, alias: &RoomAliasId, room_id: &RoomId, user_id: &UserId) -> Result<()> { if alias == services().globals.admin_alias && user_id != services().globals.server_user { diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index 6b14fda0..ca04b1e5 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -5,9 +5,8 @@ use std::{ sync::Arc, }; -use conduit::{debug, error, trace, warn, Error, Result, Server}; +use conduit::{debug, error, trace, warn, Error, Result}; use data::Data; -use database::Database; use ruma::{api::client::error::ErrorKind, EventId, RoomId}; use crate::services; @@ -16,13 +15,17 @@ pub struct Service { db: Data, } -impl Service { - pub fn build(server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(server, db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.server, args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { pub async fn event_ids_iter<'a>( &self, room_id: &RoomId, starting_events_: Vec>, ) -> Result> + 'a> { diff --git a/src/service/rooms/directory/mod.rs b/src/service/rooms/directory/mod.rs index 87c7cf92..3e60831c 100644 --- a/src/service/rooms/directory/mod.rs +++ b/src/service/rooms/directory/mod.rs @@ -2,9 +2,7 @@ mod data; use std::sync::Arc; -use conduit::Server; use data::Data; -use database::Database; use ruma::{OwnedRoomId, RoomId}; use crate::Result; @@ -13,13 +11,17 @@ pub struct Service { db: Data, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { #[tracing::instrument(skip(self))] pub fn set_public(&self, room_id: &RoomId) -> Result<()> { self.db.set_public(room_id) } diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index ed09e2e6..bb22c6c8 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -9,8 +9,7 @@ use std::{ time::{Duration, Instant}, }; -use conduit::{debug_error, debug_info, Error, Result, Server}; -use database::Database; +use conduit::{debug_error, debug_info, Error, Result}; use futures_util::Future; pub use parse_incoming_pdu::parse_incoming_pdu; use ruma::{ @@ -45,9 +44,13 @@ type AsyncRecursiveCanonicalJsonVec<'a> = type AsyncRecursiveCanonicalJsonResult<'a> = AsyncRecursiveType<'a, Result<(Arc, BTreeMap)>>; -impl Service { - pub fn build(_server: &Arc, _db: &Arc) -> Result { Ok(Self {}) } +impl crate::Service for Service { + fn build(_args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self {})) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { /// When receiving an event one needs to: /// 0. Check the server is in the room /// 1. Skip the PDU if we already know about it @@ -180,7 +183,7 @@ impl Service { .globals .bad_event_ratelimiter .write() - .await + .expect("locked") .entry((*prev_id).to_owned()) { hash_map::Entry::Vacant(e) => { @@ -200,7 +203,7 @@ impl Service { .globals .roomid_federationhandletime .write() - .await + .expect("locked") .insert(room_id.to_owned(), (event_id.to_owned(), start_time)); let r = self @@ -211,7 +214,7 @@ impl Service { .globals .roomid_federationhandletime .write() - .await + .expect("locked") .remove(&room_id.to_owned()); r @@ -245,7 +248,7 @@ impl Service { .globals .bad_event_ratelimiter .read() - .await + .expect("locked") .get(prev_id) { // Exponential backoff @@ -274,7 +277,7 @@ impl Service { .globals .roomid_federationhandletime .write() - .await + .expect("locked") .insert(room_id.to_owned(), ((*prev_id).to_owned(), start_time)); self.upgrade_outlier_to_timeline_pdu(pdu, json, create_event, origin, room_id, pub_key_map) @@ -284,7 +287,7 @@ impl Service { .globals .roomid_federationhandletime .write() - .await + .expect("locked") .remove(&room_id.to_owned()); debug!( @@ -1043,7 +1046,7 @@ impl Service { .globals .bad_event_ratelimiter .write() - .await + .expect("locked") .entry(id) { hash_map::Entry::Vacant(e) => { @@ -1076,7 +1079,7 @@ impl Service { .globals .bad_event_ratelimiter .read() - .await + .expect("locked") .get(&*next_id) { // Exponential backoff @@ -1184,7 +1187,7 @@ impl Service { .globals .bad_event_ratelimiter .read() - .await + .expect("locked") .get(&**next_id) { // Exponential backoff diff --git a/src/service/rooms/lazy_loading/mod.rs b/src/service/rooms/lazy_loading/mod.rs index 1f2ae6dd..185cfd8c 100644 --- a/src/service/rooms/lazy_loading/mod.rs +++ b/src/service/rooms/lazy_loading/mod.rs @@ -2,14 +2,12 @@ mod data; use std::{ collections::{HashMap, HashSet}, - sync::Arc, + fmt::Write, + sync::{Arc, Mutex}, }; -use conduit::Server; use data::Data; -use database::Database; use ruma::{DeviceId, OwnedDeviceId, OwnedRoomId, OwnedUserId, RoomId, UserId}; -use tokio::sync::Mutex; use crate::{PduCount, Result}; @@ -20,14 +18,27 @@ pub struct Service { pub lazy_load_waiting: Mutex>>, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), lazy_load_waiting: Mutex::new(HashMap::new()), - }) + })) } + fn memory_usage(&self, out: &mut dyn Write) -> Result<()> { + let lazy_load_waiting = self.lazy_load_waiting.lock().expect("locked").len(); + writeln!(out, "lazy_load_waiting: {lazy_load_waiting}")?; + + Ok(()) + } + + fn clear_cache(&self) { self.lazy_load_waiting.lock().expect("locked").clear(); } + + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { #[tracing::instrument(skip(self))] pub fn lazy_load_was_sent_before( &self, user_id: &UserId, device_id: &DeviceId, room_id: &RoomId, ll_user: &UserId, @@ -43,7 +54,7 @@ impl Service { ) { self.lazy_load_waiting .lock() - .await + .expect("locked") .insert((user_id.to_owned(), device_id.to_owned(), room_id.to_owned(), count), lazy_load); } @@ -51,7 +62,7 @@ impl Service { pub async fn lazy_load_confirm_delivery( &self, user_id: &UserId, device_id: &DeviceId, room_id: &RoomId, since: PduCount, ) -> Result<()> { - if let Some(user_ids) = self.lazy_load_waiting.lock().await.remove(&( + if let Some(user_ids) = self.lazy_load_waiting.lock().expect("locked").remove(&( user_id.to_owned(), device_id.to_owned(), room_id.to_owned(), diff --git a/src/service/rooms/metadata/mod.rs b/src/service/rooms/metadata/mod.rs index b91fc67e..3a96b89f 100644 --- a/src/service/rooms/metadata/mod.rs +++ b/src/service/rooms/metadata/mod.rs @@ -2,22 +2,25 @@ mod data; use std::sync::Arc; -use conduit::{Result, Server}; +use conduit::Result; use data::Data; -use database::Database; use ruma::{OwnedRoomId, RoomId}; pub struct Service { db: Data, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { /// Checks if a room exists. #[tracing::instrument(skip(self))] pub fn exists(&self, room_id: &RoomId) -> Result { self.db.exists(room_id) } diff --git a/src/service/rooms/mod.rs b/src/service/rooms/mod.rs index bef56a25..ef50b094 100644 --- a/src/service/rooms/mod.rs +++ b/src/service/rooms/mod.rs @@ -19,25 +19,27 @@ pub mod timeline; pub mod typing; pub mod user; +use std::sync::Arc; + pub struct Service { - pub alias: alias::Service, - pub auth_chain: auth_chain::Service, - pub directory: directory::Service, - pub event_handler: event_handler::Service, - pub lazy_loading: lazy_loading::Service, - pub metadata: metadata::Service, - pub outlier: outlier::Service, - pub pdu_metadata: pdu_metadata::Service, - pub read_receipt: read_receipt::Service, - pub search: search::Service, - pub short: short::Service, - pub state: state::Service, - pub state_accessor: state_accessor::Service, - pub state_cache: state_cache::Service, - pub state_compressor: state_compressor::Service, - pub timeline: timeline::Service, - pub threads: threads::Service, - pub typing: typing::Service, - pub spaces: spaces::Service, - pub user: user::Service, + pub alias: Arc, + pub auth_chain: Arc, + pub directory: Arc, + pub event_handler: Arc, + pub lazy_loading: Arc, + pub metadata: Arc, + pub outlier: Arc, + pub pdu_metadata: Arc, + pub read_receipt: Arc, + pub search: Arc, + pub short: Arc, + pub state: Arc, + pub state_accessor: Arc, + pub state_cache: Arc, + pub state_compressor: Arc, + pub timeline: Arc, + pub threads: Arc, + pub typing: Arc, + pub spaces: Arc, + pub user: Arc, } diff --git a/src/service/rooms/outlier/mod.rs b/src/service/rooms/outlier/mod.rs index 0da12a14..a7326f42 100644 --- a/src/service/rooms/outlier/mod.rs +++ b/src/service/rooms/outlier/mod.rs @@ -2,9 +2,8 @@ mod data; use std::sync::Arc; -use conduit::{Result, Server}; +use conduit::Result; use data::Data; -use database::Database; use ruma::{CanonicalJsonObject, EventId}; use crate::PduEvent; @@ -13,13 +12,17 @@ pub struct Service { db: Data, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { /// Returns the pdu from the outlier tree. pub fn get_outlier_pdu_json(&self, event_id: &EventId) -> Result> { self.db.get_outlier_pdu_json(event_id) diff --git a/src/service/rooms/pdu_metadata/mod.rs b/src/service/rooms/pdu_metadata/mod.rs index 978a0ec0..e35c969d 100644 --- a/src/service/rooms/pdu_metadata/mod.rs +++ b/src/service/rooms/pdu_metadata/mod.rs @@ -2,9 +2,8 @@ mod data; use std::sync::Arc; -use conduit::{Result, Server}; +use conduit::Result; use data::Data; -use database::Database; use ruma::{ api::{client::relations::get_relating_events, Direction}, events::{relation::RelationType, TimelineEventType}, @@ -28,13 +27,17 @@ struct ExtractRelatesToEventId { relates_to: ExtractRelType, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { #[tracing::instrument(skip(self, from, to))] pub fn add_relation(&self, from: PduCount, to: PduCount) -> Result<()> { match (from, to) { diff --git a/src/service/rooms/read_receipt/mod.rs b/src/service/rooms/read_receipt/mod.rs index ccc17d3a..e46027b7 100644 --- a/src/service/rooms/read_receipt/mod.rs +++ b/src/service/rooms/read_receipt/mod.rs @@ -2,9 +2,8 @@ mod data; use std::sync::Arc; -use conduit::{Result, Server}; +use conduit::Result; use data::Data; -use database::Database; use ruma::{events::receipt::ReceiptEvent, serde::Raw, OwnedUserId, RoomId, UserId}; use crate::services; @@ -13,13 +12,17 @@ pub struct Service { db: Data, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { /// Replaces the previous read receipt. pub fn readreceipt_update(&self, user_id: &UserId, room_id: &RoomId, event: &ReceiptEvent) -> Result<()> { self.db.readreceipt_update(user_id, room_id, event)?; diff --git a/src/service/rooms/search/mod.rs b/src/service/rooms/search/mod.rs index e2ef0f80..7573d218 100644 --- a/src/service/rooms/search/mod.rs +++ b/src/service/rooms/search/mod.rs @@ -2,22 +2,25 @@ mod data; use std::sync::Arc; -use conduit::{Result, Server}; +use conduit::Result; use data::Data; -use database::Database; use ruma::RoomId; pub struct Service { db: Data, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { #[tracing::instrument(skip(self))] pub fn index_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) -> Result<()> { self.db.index_pdu(shortroomid, pdu_id, message_body) diff --git a/src/service/rooms/short/mod.rs b/src/service/rooms/short/mod.rs index 638b6c1c..0979fb4f 100644 --- a/src/service/rooms/short/mod.rs +++ b/src/service/rooms/short/mod.rs @@ -2,22 +2,25 @@ mod data; use std::sync::Arc; -use conduit::{Result, Server}; +use conduit::Result; use data::Data; -use database::Database; use ruma::{events::StateEventType, EventId, RoomId}; pub struct Service { db: Data, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { pub fn get_or_create_shorteventid(&self, event_id: &EventId) -> Result { self.db.get_or_create_shorteventid(event_id) } diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index 6a87ac98..bf6cc873 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -7,8 +7,7 @@ use std::{ sync::Arc, }; -use conduit::{debug_info, Server}; -use database::Database; +use conduit::debug_info; use lru_cache::LruCache; use ruma::{ api::{ @@ -159,17 +158,21 @@ impl From for SpaceHierarchyRoomsChunk { } } -impl Service { - pub fn build(server: &Arc, _db: &Arc) -> Result { - let config = &server.config; - Ok(Self { +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + let config = &args.server.config; + Ok(Arc::new(Self { roomid_spacehierarchy_cache: Mutex::new(LruCache::new( (f64::from(config.roomid_spacehierarchy_cache_capacity) * config.conduit_cache_capacity_modifier) as usize, )), - }) + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { /// Gets the response for the space hierarchy over federation request /// /// Errors if the room does not exist, so a check if the room exists should diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index 42483c97..bf3b29b7 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -7,10 +7,9 @@ use std::{ use conduit::{ utils::{calculate_hash, mutex_map}, - warn, Error, Result, Server, + warn, Error, Result, }; use data::Data; -use database::Database; use ruma::{ api::client::error::ErrorKind, events::{ @@ -29,13 +28,17 @@ pub struct Service { db: Data, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { /// Set the room to the given statehash and update caches. pub async fn force_state( &self, diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 03e22187..ac127b35 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -2,12 +2,12 @@ mod data; use std::{ collections::HashMap, + fmt::Write, sync::{Arc, Mutex as StdMutex, Mutex}, }; -use conduit::{error, utils::mutex_map, warn, Error, Result, Server}; +use conduit::{error, utils::mutex_map, warn, Error, Result}; use data::Data; -use database::Database; use lru_cache::LruCache; use ruma::{ events::{ @@ -41,20 +41,39 @@ pub struct Service { pub user_visibility_cache: Mutex>, } -impl Service { - pub fn build(server: &Arc, db: &Arc) -> Result { - let config = &server.config; - Ok(Self { - db: Data::new(db), +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + let config = &args.server.config; + Ok(Arc::new(Self { + db: Data::new(args.db), server_visibility_cache: StdMutex::new(LruCache::new( (f64::from(config.server_visibility_cache_capacity) * config.conduit_cache_capacity_modifier) as usize, )), user_visibility_cache: StdMutex::new(LruCache::new( (f64::from(config.user_visibility_cache_capacity) * config.conduit_cache_capacity_modifier) as usize, )), - }) + })) } + fn memory_usage(&self, out: &mut dyn Write) -> Result<()> { + let server_visibility_cache = self.server_visibility_cache.lock().expect("locked").len(); + writeln!(out, "server_visibility_cache: {server_visibility_cache}")?; + + let user_visibility_cache = self.user_visibility_cache.lock().expect("locked").len(); + writeln!(out, "user_visibility_cache: {user_visibility_cache}")?; + + Ok(()) + } + + fn clear_cache(&self) { + self.server_visibility_cache.lock().expect("locked").clear(); + self.user_visibility_cache.lock().expect("locked").clear(); + } + + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { /// Builds a StateMap by iterating over all keys that start /// with state_hash, this gives the full state for the given state_hash. #[tracing::instrument(skip(self))] diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index 5038ef1c..45e05c29 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -2,9 +2,8 @@ mod data; use std::sync::Arc; -use conduit::{error, warn, Error, Result, Server}; +use conduit::{error, warn, Error, Result}; use data::Data; -use database::Database; use itertools::Itertools; use ruma::{ events::{ @@ -28,13 +27,17 @@ pub struct Service { db: Data, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { /// Update current membership data. #[tracing::instrument(skip(self, last_state))] #[allow(clippy::too_many_arguments)] diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index 08112995..564a0bee 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -2,13 +2,13 @@ mod data; use std::{ collections::HashSet, + fmt::Write, mem::size_of, sync::{Arc, Mutex as StdMutex, Mutex}, }; -use conduit::{utils, Result, Server}; +use conduit::{utils, Result}; use data::Data; -use database::Database; use lru_cache::LruCache; use ruma::{EventId, RoomId}; @@ -52,17 +52,30 @@ pub struct Service { pub stateinfo_cache: StateInfoLruCache, } -impl Service { - pub fn build(server: &Arc, db: &Arc) -> Result { - let config = &server.config; - Ok(Self { - db: Data::new(db), +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + let config = &args.server.config; + Ok(Arc::new(Self { + db: Data::new(args.db), stateinfo_cache: StdMutex::new(LruCache::new( (f64::from(config.stateinfo_cache_capacity) * config.conduit_cache_capacity_modifier) as usize, )), - }) + })) } + fn memory_usage(&self, out: &mut dyn Write) -> Result<()> { + let stateinfo_cache = self.stateinfo_cache.lock().expect("locked").len(); + writeln!(out, "stateinfo_cache: {stateinfo_cache}")?; + + Ok(()) + } + + fn clear_cache(&self) { self.stateinfo_cache.lock().expect("locked").clear(); } + + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { /// Returns a stack with info on shortstatehash, full state, added diff and /// removed diff for the selected shortstatehash and each parent layer. #[tracing::instrument(skip(self))] diff --git a/src/service/rooms/threads/mod.rs b/src/service/rooms/threads/mod.rs index f47fc499..f3cefe21 100644 --- a/src/service/rooms/threads/mod.rs +++ b/src/service/rooms/threads/mod.rs @@ -2,9 +2,8 @@ mod data; use std::{collections::BTreeMap, sync::Arc}; -use conduit::{Error, Result, Server}; +use conduit::{Error, Result}; use data::Data; -use database::Database; use ruma::{ api::client::{error::ErrorKind, threads::get_threads::v1::IncludeThreads}, events::relation::BundledThread, @@ -18,13 +17,17 @@ pub struct Service { db: Data, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { pub fn threads_until<'a>( &'a self, user_id: &'a UserId, room_id: &'a RoomId, until: u64, include: &'a IncludeThreads, ) -> Result> + 'a> { diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index dfcfedea..f7e8384b 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -2,12 +2,12 @@ mod data; use std::{ collections::{BTreeMap, HashSet}, + fmt::Write, sync::Arc, }; -use conduit::{debug, error, info, utils, utils::mutex_map, warn, Error, Result, Server}; +use conduit::{debug, error, info, utils, utils::mutex_map, warn, Error, Result}; use data::Data; -use database::Database; use itertools::Itertools; use rand::prelude::SliceRandom; use ruma::{ @@ -68,13 +68,37 @@ pub struct Service { db: Data, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn memory_usage(&self, out: &mut dyn Write) -> Result<()> { + let lasttimelinecount_cache = self + .db + .lasttimelinecount_cache + .lock() + .expect("locked") + .len(); + writeln!(out, "lasttimelinecount_cache: {lasttimelinecount_cache}")?; + + Ok(()) + } + + fn clear_cache(&self) { + self.db + .lasttimelinecount_cache + .lock() + .expect("locked") + .clear(); + } + + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { #[tracing::instrument(skip(self))] pub fn first_pdu_in_room(&self, room_id: &RoomId) -> Result>> { self.all_pdus(user_id!("@doesntmatter:conduit.rs"), room_id)? @@ -1238,19 +1262,6 @@ impl Service { debug!("Prepended backfill pdu"); Ok(()) } - - pub fn get_lasttimelinecount_cache_usage(&self) -> (usize, usize) { - let cache = self.db.lasttimelinecount_cache.lock().expect("locked"); - (cache.len(), cache.capacity()) - } - - pub fn clear_lasttimelinecount_cache(&self) { - self.db - .lasttimelinecount_cache - .lock() - .expect("locked") - .clear(); - } } #[cfg(test)] diff --git a/src/service/rooms/typing/mod.rs b/src/service/rooms/typing/mod.rs index dd4e7fe2..715e3162 100644 --- a/src/service/rooms/typing/mod.rs +++ b/src/service/rooms/typing/mod.rs @@ -1,7 +1,6 @@ use std::{collections::BTreeMap, sync::Arc}; -use conduit::{debug_info, trace, utils, Result, Server}; -use database::Database; +use conduit::{debug_info, trace, utils, Result}; use ruma::{ api::federation::transactions::edu::{Edu, TypingContent}, events::SyncEphemeralRoomEvent, @@ -19,15 +18,19 @@ pub struct Service { pub typing_update_sender: broadcast::Sender, } -impl Service { - pub fn build(_server: &Arc, _db: &Arc) -> Result { - Ok(Self { +impl crate::Service for Service { + fn build(_args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { typing: RwLock::new(BTreeMap::new()), last_typing_update: RwLock::new(BTreeMap::new()), typing_update_sender: broadcast::channel(100).0, - }) + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { /// Sets a user as typing until the timeout timestamp is reached or /// roomtyping_remove is called. pub async fn typing_add(&self, user_id: &UserId, room_id: &RoomId, timeout: u64) -> Result<()> { diff --git a/src/service/rooms/user/mod.rs b/src/service/rooms/user/mod.rs index 505e0662..12124a57 100644 --- a/src/service/rooms/user/mod.rs +++ b/src/service/rooms/user/mod.rs @@ -2,22 +2,25 @@ mod data; use std::sync::Arc; -use conduit::{Result, Server}; +use conduit::Result; use data::Data; -use database::Database; use ruma::{OwnedRoomId, OwnedUserId, RoomId, UserId}; pub struct Service { db: Data, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { pub fn reset_notification_counts(&self, user_id: &UserId, room_id: &RoomId) -> Result<()> { self.db.reset_notification_counts(user_id, room_id) } diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index c57ca14d..b75ca8b4 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -6,9 +6,9 @@ mod sender; use std::{fmt::Debug, sync::Arc}; -use conduit::{Error, Result, Server}; +use async_trait::async_trait; +use conduit::{Error, Result}; use data::Data; -use database::Database; pub use resolve::FedDest; use ruma::{ api::{appservice::Registration, OutgoingRequest}, @@ -53,12 +53,13 @@ pub enum SendingEvent { Flush, // none } -impl Service { - pub fn build(server: &Arc, db: &Arc) -> Result> { - let config = &server.config; +#[async_trait] +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + let config = &args.server.config; let (sender, receiver) = loole::unbounded(); Ok(Arc::new(Self { - db: Data::new(db.clone()), + db: Data::new(args.db.clone()), sender, receiver: Mutex::new(receiver), handler_join: Mutex::new(None), @@ -67,7 +68,13 @@ impl Service { })) } - pub async fn close(&self) { + async fn start(self: Arc) -> Result<()> { + self.start_handler().await; + + Ok(()) + } + + async fn stop(&self) { self.interrupt(); if let Some(handler_join) = self.handler_join.lock().await.take() { if let Err(e) = handler_join.await { @@ -76,12 +83,16 @@ impl Service { } } - pub fn interrupt(&self) { + fn interrupt(&self) { if !self.sender.is_closed() { self.sender.close(); } } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { #[tracing::instrument(skip(self, pdu_id, user, pushkey))] pub fn send_pdu_push(&self, pdu_id: &[u8], user: &UserId, pushkey: String) -> Result<()> { let dest = Destination::Push(user.to_owned(), pushkey); diff --git a/src/service/service.rs b/src/service/service.rs new file mode 100644 index 00000000..ef60f359 --- /dev/null +++ b/src/service/service.rs @@ -0,0 +1,53 @@ +use std::{collections::BTreeMap, fmt::Write, sync::Arc}; + +use async_trait::async_trait; +use conduit::{utils::string::split_once_infallible, Result, Server}; +use database::Database; + +#[async_trait] +pub(crate) trait Service: Send + Sync { + /// Implement the construction of the service instance. Services are + /// generally singletons so expect this to only be called once for a + /// service type. Note that it may be called again after a server reload, + /// but the prior instance will have been dropped first. Failure will + /// shutdown the server with an error. + fn build(args: Args<'_>) -> Result> + where + Self: Sized; + + /// Start the service. Implement the spawning of any service workers. This + /// is called after all other services have been constructed. Failure will + /// shutdown the server with an error. + async fn start(self: Arc) -> Result<()> { Ok(()) } + + /// Stop the service. Implement the joining of any service workers and + /// cleanup of any other state. This function is asynchronous to allow that + /// gracefully, but errors cannot propagate. + async fn stop(&self) {} + + /// Interrupt the service. This may be sent prior to `stop()` as a + /// notification to improve the shutdown sequence. Implementations must be + /// robust to this being called multiple times. + fn interrupt(&self) {} + + /// Clear any caches or similar runtime state. + fn clear_cache(&self) {} + + /// Memory usage report in a markdown string. + fn memory_usage(&self, _out: &mut dyn Write) -> Result<()> { Ok(()) } + + /// Return the name of the service. + /// i.e. `crate::service::make_name(std::module_path!())` + fn name(&self) -> &str; +} + +pub(crate) struct Args<'a> { + pub(crate) server: &'a Arc, + pub(crate) db: &'a Arc, + pub(crate) _service: &'a Map, +} + +pub(crate) type Map = BTreeMap>; + +#[inline] +pub(crate) fn make_name(module_path: &str) -> &str { split_once_infallible(module_path, "::").1 } diff --git a/src/service/services.rs b/src/service/services.rs index 81aba231..0ba86693 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -1,101 +1,97 @@ -use std::sync::Arc; +use std::{collections::BTreeMap, fmt::Write, sync::Arc}; -use conduit::{debug_info, Result, Server}; +use conduit::{debug, debug_info, info, trace, Result, Server}; use database::Database; -use tracing::{debug, info, trace}; use crate::{ - account_data, admin, appservice, globals, key_backups, media, presence, pusher, rooms, sending, transaction_ids, - uiaa, users, + account_data, admin, appservice, globals, key_backups, media, presence, pusher, rooms, sending, + service::{Args, Map, Service}, + transaction_ids, uiaa, users, }; pub struct Services { pub rooms: rooms::Service, - pub appservice: appservice::Service, - pub pusher: pusher::Service, - pub transaction_ids: transaction_ids::Service, - pub uiaa: uiaa::Service, - pub users: users::Service, - pub account_data: account_data::Service, + pub appservice: Arc, + pub pusher: Arc, + pub transaction_ids: Arc, + pub uiaa: Arc, + pub users: Arc, + pub account_data: Arc, pub presence: Arc, pub admin: Arc, - pub globals: globals::Service, - pub key_backups: key_backups::Service, - pub media: media::Service, + pub globals: Arc, + pub key_backups: Arc, + pub media: Arc, pub sending: Arc, + + pub(crate) service: Map, pub server: Arc, pub db: Arc, } impl Services { - pub async fn build(server: Arc, db: Arc) -> Result { + pub fn build(server: Arc, db: Arc) -> Result { + let mut service: Map = BTreeMap::new(); + macro_rules! build { + ($tyname:ty) => {{ + let built = <$tyname>::build(Args { + server: &server, + db: &db, + _service: &service, + })?; + service.insert(built.name().to_owned(), built.clone()); + built + }}; + } + Ok(Self { rooms: rooms::Service { - alias: rooms::alias::Service::build(&server, &db)?, - auth_chain: rooms::auth_chain::Service::build(&server, &db)?, - directory: rooms::directory::Service::build(&server, &db)?, - event_handler: rooms::event_handler::Service::build(&server, &db)?, - lazy_loading: rooms::lazy_loading::Service::build(&server, &db)?, - metadata: rooms::metadata::Service::build(&server, &db)?, - outlier: rooms::outlier::Service::build(&server, &db)?, - pdu_metadata: rooms::pdu_metadata::Service::build(&server, &db)?, - read_receipt: rooms::read_receipt::Service::build(&server, &db)?, - search: rooms::search::Service::build(&server, &db)?, - short: rooms::short::Service::build(&server, &db)?, - state: rooms::state::Service::build(&server, &db)?, - state_accessor: rooms::state_accessor::Service::build(&server, &db)?, - state_cache: rooms::state_cache::Service::build(&server, &db)?, - state_compressor: rooms::state_compressor::Service::build(&server, &db)?, - timeline: rooms::timeline::Service::build(&server, &db)?, - threads: rooms::threads::Service::build(&server, &db)?, - typing: rooms::typing::Service::build(&server, &db)?, - spaces: rooms::spaces::Service::build(&server, &db)?, - user: rooms::user::Service::build(&server, &db)?, + alias: build!(rooms::alias::Service), + auth_chain: build!(rooms::auth_chain::Service), + directory: build!(rooms::directory::Service), + event_handler: build!(rooms::event_handler::Service), + lazy_loading: build!(rooms::lazy_loading::Service), + metadata: build!(rooms::metadata::Service), + outlier: build!(rooms::outlier::Service), + pdu_metadata: build!(rooms::pdu_metadata::Service), + read_receipt: build!(rooms::read_receipt::Service), + search: build!(rooms::search::Service), + short: build!(rooms::short::Service), + state: build!(rooms::state::Service), + state_accessor: build!(rooms::state_accessor::Service), + state_cache: build!(rooms::state_cache::Service), + state_compressor: build!(rooms::state_compressor::Service), + timeline: build!(rooms::timeline::Service), + threads: build!(rooms::threads::Service), + typing: build!(rooms::typing::Service), + spaces: build!(rooms::spaces::Service), + user: build!(rooms::user::Service), }, - appservice: appservice::Service::build(&server, &db)?, - pusher: pusher::Service::build(&server, &db)?, - transaction_ids: transaction_ids::Service::build(&server, &db)?, - uiaa: uiaa::Service::build(&server, &db)?, - users: users::Service::build(&server, &db)?, - account_data: account_data::Service::build(&server, &db)?, - presence: presence::Service::build(&server, &db)?, - admin: admin::Service::build(&server, &db)?, - key_backups: key_backups::Service::build(&server, &db)?, - media: media::Service::build(&server, &db)?, - sending: sending::Service::build(&server, &db)?, - globals: globals::Service::build(&server, &db)?, + appservice: build!(appservice::Service), + pusher: build!(pusher::Service), + transaction_ids: build!(transaction_ids::Service), + uiaa: build!(uiaa::Service), + users: build!(users::Service), + account_data: build!(account_data::Service), + presence: build!(presence::Service), + admin: build!(admin::Service), + key_backups: build!(key_backups::Service), + media: build!(media::Service), + sending: build!(sending::Service), + globals: build!(globals::Service), + service, server, db, }) } - pub async fn memory_usage(&self) -> String { - let lazy_load_waiting = self.rooms.lazy_loading.lazy_load_waiting.lock().await.len(); - let server_visibility_cache = self - .rooms - .state_accessor - .server_visibility_cache - .lock() - .unwrap() - .len(); - let user_visibility_cache = self - .rooms - .state_accessor - .user_visibility_cache - .lock() - .unwrap() - .len(); - let stateinfo_cache = self - .rooms - .state_compressor - .stateinfo_cache - .lock() - .unwrap() - .len(); - let lasttimelinecount_cache = self - .rooms - .timeline - .get_lasttimelinecount_cache_usage().0; + pub async fn memory_usage(&self) -> Result { + let mut out = String::new(); + for service in self.service.values() { + service.memory_usage(&mut out)?; + } + + //TODO let roomid_spacehierarchy_cache = self .rooms .spaces @@ -103,113 +99,23 @@ impl Services { .lock() .await .len(); - let resolver_overrides_cache = self - .globals - .resolver - .overrides - .read() - .expect("locked for reading") - .len(); - let resolver_destinations_cache = self - .globals - .resolver - .destinations - .read() - .expect("locked for reading") - .len(); - let bad_event_ratelimiter = self.globals.bad_event_ratelimiter.read().await.len(); - let bad_query_ratelimiter = self.globals.bad_query_ratelimiter.read().await.len(); - let bad_signature_ratelimiter = self.globals.bad_signature_ratelimiter.read().await.len(); + writeln!(out, "roomid_spacehierarchy_cache: {roomid_spacehierarchy_cache}")?; - format!( - "\ -lazy_load_waiting: {lazy_load_waiting} -server_visibility_cache: {server_visibility_cache} -user_visibility_cache: {user_visibility_cache} -stateinfo_cache: {stateinfo_cache} -lasttimelinecount_cache: {lasttimelinecount_cache} -roomid_spacehierarchy_cache: {roomid_spacehierarchy_cache} -resolver_overrides_cache: {resolver_overrides_cache} -resolver_destinations_cache: {resolver_destinations_cache} -bad_event_ratelimiter: {bad_event_ratelimiter} -bad_query_ratelimiter: {bad_query_ratelimiter} -bad_signature_ratelimiter: {bad_signature_ratelimiter} -" - ) + Ok(out) } - pub async fn clear_caches(&self, amount: u32) { - if amount > 0 { - self.rooms - .lazy_loading - .lazy_load_waiting - .lock() - .await - .clear(); - } - if amount > 1 { - self.rooms - .state_accessor - .server_visibility_cache - .lock() - .unwrap() - .clear(); - } - if amount > 2 { - self.rooms - .state_accessor - .user_visibility_cache - .lock() - .unwrap() - .clear(); - } - if amount > 3 { - self.rooms - .state_compressor - .stateinfo_cache - .lock() - .unwrap() - .clear(); - } - if amount > 4 { - self.rooms - .timeline - .clear_lasttimelinecount_cache(); - } - if amount > 5 { - self.rooms - .spaces - .roomid_spacehierarchy_cache - .lock() - .await - .clear(); - } - if amount > 6 { - self.globals - .resolver - .overrides - .write() - .expect("locked for writing") - .clear(); - self.globals - .resolver - .destinations - .write() - .expect("locked for writing") - .clear(); - } - if amount > 7 { - self.globals.resolver.resolver.clear_cache(); - } - if amount > 8 { - self.globals.bad_event_ratelimiter.write().await.clear(); - } - if amount > 9 { - self.globals.bad_query_ratelimiter.write().await.clear(); - } - if amount > 10 { - self.globals.bad_signature_ratelimiter.write().await.clear(); + pub async fn clear_cache(&self) { + for service in self.service.values() { + service.clear_cache(); } + + //TODO + self.rooms + .spaces + .roomid_spacehierarchy_cache + .lock() + .await + .clear(); } pub async fn start(&self) -> Result<()> { @@ -219,10 +125,9 @@ bad_signature_ratelimiter: {bad_signature_ratelimiter} globals::migrations::migrations(&self.db, &self.globals.config).await?; globals::emerg_access::init_emergency_access(); - self.admin.start_handler().await; - self.sending.start_handler().await; - if self.globals.config.allow_local_presence { - self.presence.start_handler().await; + for (name, service) in &self.service { + debug!("Starting {name}"); + service.clone().start().await?; } if self.globals.allow_check_for_updates() { @@ -238,18 +143,18 @@ bad_signature_ratelimiter: {bad_signature_ratelimiter} Ok(()) } - pub async fn interrupt(&self) { - trace!("Interrupting services..."); - self.sending.interrupt(); - self.presence.interrupt(); - self.admin.interrupt(); + pub fn interrupt(&self) { + debug!("Interrupting services..."); - trace!("Services interrupt complete."); + for (name, service) in &self.service { + trace!("Interrupting {name}"); + service.interrupt(); + } } pub async fn stop(&self) { info!("Shutting down services"); - self.interrupt().await; + self.interrupt(); debug!("Waiting for update worker..."); if let Some(updates_handle) = self.globals.updates_handle.lock().await.take() { @@ -261,14 +166,10 @@ bad_signature_ratelimiter: {bad_signature_ratelimiter} } } - debug!("Waiting for admin worker..."); - self.admin.close().await; - - debug!("Waiting for presence worker..."); - self.presence.close().await; - - debug!("Waiting for sender..."); - self.sending.close().await; + for (name, service) in &self.service { + debug!("Waiting for {name} ..."); + service.stop().await; + } debug_info!("Services shutdown complete."); } diff --git a/src/service/transaction_ids/mod.rs b/src/service/transaction_ids/mod.rs index f18bbe64..78e6337f 100644 --- a/src/service/transaction_ids/mod.rs +++ b/src/service/transaction_ids/mod.rs @@ -2,22 +2,25 @@ mod data; use std::sync::Arc; -use conduit::{Result, Server}; +use conduit::Result; use data::Data; -use database::Database; use ruma::{DeviceId, TransactionId, UserId}; pub struct Service { pub db: Data, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { pub fn add_txnid( &self, user_id: &UserId, device_id: Option<&DeviceId>, txn_id: &TransactionId, data: &[u8], ) -> Result<()> { diff --git a/src/service/uiaa/mod.rs b/src/service/uiaa/mod.rs index 0d2f8bf7..4b953ffb 100644 --- a/src/service/uiaa/mod.rs +++ b/src/service/uiaa/mod.rs @@ -2,9 +2,8 @@ mod data; use std::sync::Arc; -use conduit::{utils, utils::hash, Error, Result, Server}; +use conduit::{utils, utils::hash, Error, Result}; use data::Data; -use database::Database; use ruma::{ api::client::{ error::ErrorKind, @@ -22,13 +21,17 @@ pub struct Service { pub db: Data, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { /// Creates a new Uiaa session. Make sure the session token is unique. pub fn create( &self, user_id: &UserId, device_id: &DeviceId, uiaainfo: &UiaaInfo, json_body: &CanonicalJsonValue, diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index 8444d538..cd580994 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -6,9 +6,8 @@ use std::{ sync::{Arc, Mutex, Mutex as StdMutex}, }; -use conduit::{Error, Result, Server}; +use conduit::{Error, Result}; use data::Data; -use database::Database; use ruma::{ api::client::{ device::Device, @@ -41,14 +40,18 @@ pub struct Service { pub connections: DbConnections, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db.clone()), +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db.clone()), connections: StdMutex::new(BTreeMap::new()), - }) + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { /// Check if a user has an account on this homeserver. pub fn exists(&self, user_id: &UserId) -> Result { self.db.exists(user_id) } From 08274150e513482f442ce5a84a5908bd47f3a6ab Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 3 Jul 2024 09:44:52 +0000 Subject: [PATCH 007/158] cork database during global counter increment Signed-off-by: Jason Volk --- src/service/globals/data.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index 43614854..7c430a81 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -57,6 +57,7 @@ impl Data { } pub fn next_count(&self) -> Result { + let _cork = self.db.cork(); let mut lock = self.counter.write().expect("locked"); let counter: &mut u64 = &mut lock; debug_assert!( From db46d6dd6b5db90b26302749df6ed99bc0404756 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 3 Jul 2024 19:13:49 +0000 Subject: [PATCH 008/158] optimize millis_since_unix_epoch() Signed-off-by: Jason Volk --- src/core/utils/mod.rs | 2 +- src/core/utils/time.rs | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 75f55c9a..b34611df 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -23,7 +23,7 @@ pub use mutex_map::MutexMap; pub use rand::string as random_string; pub use string::{str_from_bytes, string_from_bytes}; pub use sys::available_parallelism; -pub use time::millis_since_unix_epoch; +pub use time::now_millis as millis_since_unix_epoch; use crate::Result; diff --git a/src/core/utils/time.rs b/src/core/utils/time.rs index d7cab1fb..f6208b31 100644 --- a/src/core/utils/time.rs +++ b/src/core/utils/time.rs @@ -1,12 +1,12 @@ -use std::time::{SystemTime, UNIX_EPOCH}; - #[inline] #[must_use] #[allow(clippy::as_conversions)] -pub fn millis_since_unix_epoch() -> u64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("time is valid") +pub fn now_millis() -> u64 { + use std::time::UNIX_EPOCH; + + UNIX_EPOCH + .elapsed() + .expect("positive duration after epoch") .as_millis() as u64 } From 1e8b8cce0f6ac62f4dc00ed904b55497e99f79a4 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 3 Jul 2024 20:04:15 +0000 Subject: [PATCH 009/158] even more byte utils optimizations Signed-off-by: Jason Volk --- src/core/utils/bytes.rs | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/core/utils/bytes.rs b/src/core/utils/bytes.rs index 6ebf4c4f..e8975a49 100644 --- a/src/core/utils/bytes.rs +++ b/src/core/utils/bytes.rs @@ -3,27 +3,28 @@ use crate::Result; #[inline] #[must_use] pub fn increment(old: Option<&[u8]>) -> [u8; 8] { - old.map(TryInto::try_into) - .map_or(0_u64, |val| val.map_or(0_u64, u64::from_be_bytes)) + old.map_or(0_u64, u64_from_bytes_or_zero) .wrapping_add(1) .to_be_bytes() } -/// Parses the big-endian bytes into an u64. -#[inline] -pub fn u64_from_bytes(bytes: &[u8]) -> Result { - let array: [u8; 8] = bytes.try_into()?; - Ok(u64_from_u8x8(array)) -} - -/// Parses the 8 big-endian bytes into an u64. +/// Parses 8 big-endian bytes into an u64; panic on invalid argument #[inline] #[must_use] -pub fn u64_from_u8(bytes: &[u8]) -> u64 { - let bytes: &[u8; 8] = bytes.try_into().expect("must slice at least 8 bytes"); - u64_from_u8x8(*bytes) -} +pub fn u64_from_u8(bytes: &[u8]) -> u64 { u64_from_bytes(bytes).expect("must slice at least 8 bytes") } + +/// Parses the big-endian bytes into an u64. +#[inline] +#[must_use] +pub fn u64_from_bytes_or_zero(bytes: &[u8]) -> u64 { u64_from_bytes(bytes).unwrap_or(0) } + +/// Parses the big-endian bytes into an u64. +#[inline] +pub fn u64_from_bytes(bytes: &[u8]) -> Result { Ok(u64_from_u8x8(*u8x8_from_bytes(bytes)?)) } #[inline] #[must_use] pub fn u64_from_u8x8(bytes: [u8; 8]) -> u64 { u64::from_be_bytes(bytes) } + +#[inline] +pub fn u8x8_from_bytes(bytes: &[u8]) -> Result<&[u8; 8]> { Ok(bytes.try_into()?) } From eeda96d94a8138274d6ee344611e426ec3e3ec9f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 3 Jul 2024 20:06:43 +0000 Subject: [PATCH 010/158] inline analysis and symbol reduction; emits smaller than 64 bytes marked inline Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 2 +- src/core/error.rs | 1 + src/core/log/capture/guard.rs | 1 + src/core/log/capture/layer.rs | 2 + src/core/utils/hash.rs | 3 - src/core/utils/mod.rs | 1 - src/core/utils/string.rs | 1 - src/database/cork.rs | 1 + src/database/database.rs | 3 + src/database/engine.rs | 2 + src/database/handle.rs | 2 + src/database/map.rs | 1 + src/database/slice.rs | 9 +-- src/main/restart.rs | 1 + src/service/appservice/mod.rs | 4 ++ src/service/globals/data.rs | 15 ++++- src/service/globals/migrations.rs | 65 ++++++++++--------- src/service/globals/mod.rs | 24 +------ src/service/media/data.rs | 1 + src/service/presence/mod.rs | 2 + src/service/rooms/event_handler/mod.rs | 1 + .../rooms/event_handler/signing_keys.rs | 8 ++- src/service/rooms/metadata/data.rs | 4 ++ src/service/rooms/metadata/mod.rs | 7 +- src/service/rooms/state/data.rs | 1 + src/service/rooms/state/mod.rs | 2 + src/service/rooms/state_accessor/mod.rs | 3 + src/service/rooms/state_compressor/mod.rs | 1 + src/service/rooms/timeline/data.rs | 1 + src/service/rooms/timeline/mod.rs | 4 ++ src/service/sending/data.rs | 2 + src/service/sending/mod.rs | 4 +- src/service/sending/resolve.rs | 1 + src/service/users/data.rs | 2 + src/service/users/mod.rs | 8 +++ 35 files changed, 117 insertions(+), 73 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index c5b944bf..23197431 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -15,7 +15,7 @@ use ruma::{ events::room::message::RoomMessageEventContent, CanonicalJsonObject, EventId, OwnedRoomOrAliasId, RoomId, RoomVersionId, ServerName, }; -use service::{rooms::event_handler::parse_incoming_pdu, sending::resolve::resolve_actual_dest, services, PduEvent}; +use service::{rooms::event_handler::parse_incoming_pdu, sending::resolve_actual_dest, services, PduEvent}; use tokio::sync::RwLock; use tracing_subscriber::EnvFilter; diff --git a/src/core/error.rs b/src/core/error.rs index ac2e176d..f977803d 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -92,6 +92,7 @@ impl Error { } /// Returns the Matrix error code / error kind + #[inline] pub fn error_code(&self) -> ruma::api::client::error::ErrorKind { if let Self::Federation(_, error) = self { return error.error_kind().unwrap_or_else(|| &Unknown).clone(); diff --git a/src/core/log/capture/guard.rs b/src/core/log/capture/guard.rs index a126fedc..b5a6d8b3 100644 --- a/src/core/log/capture/guard.rs +++ b/src/core/log/capture/guard.rs @@ -8,5 +8,6 @@ pub struct Guard { } impl Drop for Guard { + #[inline] fn drop(&mut self) { self.capture.stop(); } } diff --git a/src/core/log/capture/layer.rs b/src/core/log/capture/layer.rs index 19d76771..57f22dc7 100644 --- a/src/core/log/capture/layer.rs +++ b/src/core/log/capture/layer.rs @@ -17,6 +17,7 @@ struct Visitor { } impl Layer { + #[inline] pub fn new(state: &Arc) -> Self { Self { state: state.clone(), @@ -25,6 +26,7 @@ impl Layer { } impl fmt::Debug for Layer { + #[inline] fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { formatter.debug_struct("capture::Layer").finish() } diff --git a/src/core/utils/hash.rs b/src/core/utils/hash.rs index 9469321f..5a3664cb 100644 --- a/src/core/utils/hash.rs +++ b/src/core/utils/hash.rs @@ -3,14 +3,11 @@ mod sha256; use crate::Result; -#[inline] pub fn password(password: &str) -> Result { argon::password(password) } -#[inline] pub fn verify_password(password: &str, password_hash: &str) -> Result<()> { argon::verify_password(password, password_hash) } -#[inline] #[must_use] pub fn calculate_hash(keys: &[&[u8]]) -> Vec { sha256::hash(keys) } diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index b34611df..f9f1b87e 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -33,7 +33,6 @@ pub fn clamp(val: T, min: T, max: T) -> T { cmp::min(cmp::max(val, min), /// /// * #[must_use] -#[inline(always)] pub fn unwrap_infallible(result: Result) -> T { match result { Ok(val) => val, diff --git a/src/core/utils/string.rs b/src/core/utils/string.rs index 78765154..1a5b6b72 100644 --- a/src/core/utils/string.rs +++ b/src/core/utils/string.rs @@ -9,7 +9,6 @@ pub fn split_once_infallible<'a>(input: &'a str, delim: &'_ str) -> (&'a str, &' } /// Parses the bytes into a string. -#[inline] pub fn string_from_bytes(bytes: &[u8]) -> Result { let str: &str = str_from_bytes(bytes)?; Ok(str.to_owned()) diff --git a/src/database/cork.rs b/src/database/cork.rs index b7e1f60f..26c520a2 100644 --- a/src/database/cork.rs +++ b/src/database/cork.rs @@ -9,6 +9,7 @@ pub struct Cork { } impl Cork { + #[inline] pub(super) fn new(db: &Arc, flush: bool, sync: bool) -> Self { db.cork(); Self { diff --git a/src/database/database.rs b/src/database/database.rs index 2c6c6808..06a2d88f 100644 --- a/src/database/database.rs +++ b/src/database/database.rs @@ -19,12 +19,15 @@ impl Database { }) } + #[inline] #[must_use] pub fn cork(&self) -> Cork { Cork::new(&self.db, false, false) } + #[inline] #[must_use] pub fn cork_and_flush(&self) -> Cork { Cork::new(&self.db, true, false) } + #[inline] #[must_use] pub fn cork_and_sync(&self) -> Cork { Cork::new(&self.db, true, true) } } diff --git a/src/database/engine.rs b/src/database/engine.rs index fe16d29f..7c9522e1 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -121,6 +121,7 @@ impl Engine { pub fn sync(&self) -> Result<()> { result(DBCommon::flush_wal(&self.db, true)) } + #[inline] pub fn corked(&self) -> bool { self.corks.load(std::sync::atomic::Ordering::Relaxed) > 0 } pub(crate) fn cork(&self) { @@ -242,6 +243,7 @@ impl Engine { } impl Drop for Engine { + #[cold] fn drop(&mut self) { const BLOCKING: bool = true; diff --git a/src/database/handle.rs b/src/database/handle.rs index 787a5cea..0b45a75f 100644 --- a/src/database/handle.rs +++ b/src/database/handle.rs @@ -17,9 +17,11 @@ impl<'a> From> for Handle<'a> { impl Deref for Handle<'_> { type Target = [u8]; + #[inline] fn deref(&self) -> &Self::Target { &self.val } } impl AsRef<[u8]> for Handle<'_> { + #[inline] fn as_ref(&self) -> &[u8] { &self.val } } diff --git a/src/database/map.rs b/src/database/map.rs index 73cf5107..0b007307 100644 --- a/src/database/map.rs +++ b/src/database/map.rs @@ -199,6 +199,7 @@ impl<'a> IntoIterator for &'a Map { type IntoIter = Box + Send + 'a>; type Item = OwnedKeyValPair; + #[inline] fn into_iter(self) -> Self::IntoIter { self.iter() } } diff --git a/src/database/slice.rs b/src/database/slice.rs index 770f1957..448d969d 100644 --- a/src/database/slice.rs +++ b/src/database/slice.rs @@ -11,21 +11,19 @@ pub type Key = [Byte]; pub(crate) type Byte = u8; impl OwnedKeyVal { - #[inline] #[must_use] pub fn as_slice(&self) -> KeyVal<'_> { KeyVal(&self.0, &self.1) } - #[inline] #[must_use] pub fn to_tuple(self) -> OwnedKeyValPair { (self.0, self.1) } } impl From for OwnedKeyVal { - #[inline] fn from((key, val): OwnedKeyValPair) -> Self { Self(key, val) } } impl From<&KeyVal<'_>> for OwnedKeyVal { + #[inline] fn from(slice: &KeyVal<'_>) -> Self { slice.to_owned() } } @@ -34,7 +32,6 @@ impl From> for OwnedKeyVal { } impl From for OwnedKeyValPair { - #[inline] fn from(val: OwnedKeyVal) -> Self { val.to_tuple() } } @@ -43,22 +40,18 @@ impl KeyVal<'_> { #[must_use] pub fn to_owned(&self) -> OwnedKeyVal { OwnedKeyVal::from(self) } - #[inline] #[must_use] pub fn as_tuple(&self) -> KeyValPair<'_> { (self.0, self.1) } } impl<'a> From<&'a OwnedKeyVal> for KeyVal<'a> { - #[inline] fn from(owned: &'a OwnedKeyVal) -> Self { owned.as_slice() } } impl<'a> From<&'a OwnedKeyValPair> for KeyVal<'a> { - #[inline] fn from((key, val): &'a OwnedKeyValPair) -> Self { KeyVal(key.as_slice(), val.as_slice()) } } impl<'a> From> for KeyVal<'a> { - #[inline] fn from((key, val): KeyValPair<'a>) -> Self { KeyVal(key, val) } } diff --git a/src/main/restart.rs b/src/main/restart.rs index 00de9f21..009e5228 100644 --- a/src/main/restart.rs +++ b/src/main/restart.rs @@ -4,6 +4,7 @@ use std::{env, os::unix::process::CommandExt, process::Command}; use conduit::{debug, info, utils}; +#[cold] pub(super) fn restart() -> ! { // SAFETY: We have allowed an override for the case where the current_exe() has // been replaced or removed. By default the server will fail to restart if the diff --git a/src/service/appservice/mod.rs b/src/service/appservice/mod.rs index 9bb00df1..f7cf0b6b 100644 --- a/src/service/appservice/mod.rs +++ b/src/service/appservice/mod.rs @@ -23,6 +23,7 @@ pub struct NamespaceRegex { impl NamespaceRegex { /// Checks if this namespace has rights to a namespace + #[inline] #[must_use] pub fn is_match(&self, heystack: &str) -> bool { if self.is_exclusive_match(heystack) { @@ -38,6 +39,7 @@ impl NamespaceRegex { } /// Checks if this namespace has exlusive rights to a namespace + #[inline] #[must_use] pub fn is_exclusive_match(&self, heystack: &str) -> bool { if let Some(exclusive) = &self.exclusive { @@ -55,6 +57,7 @@ impl RegistrationInfo { self.users.is_match(user_id.as_str()) || self.registration.sender_localpart == user_id.localpart() } + #[inline] #[must_use] pub fn is_exclusive_user_match(&self, user_id: &UserId) -> bool { self.users.is_exclusive_match(user_id.as_str()) || self.registration.sender_localpart == user_id.localpart() @@ -143,6 +146,7 @@ impl crate::Service for Service { } impl Service { + #[inline] pub fn all(&self) -> Result> { iter_ids(&self.db) } /// Registers an appservice and returns the ID to the caller diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index 7c430a81..ecb827fe 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -99,6 +99,7 @@ impl Data { }) } + #[inline] pub fn update_check_for_updates_id(&self, id: u64) -> Result<()> { self.global .insert(LAST_CHECK_FOR_UPDATES_COUNT, &id.to_be_bytes())?; @@ -207,8 +208,6 @@ impl Data { Ok(()) } - pub fn cleanup(&self) -> Result<()> { self.db.db.cleanup() } - pub fn load_keypair(&self) -> Result { let keypair_bytes = self.global.get(b"keypair")?.map_or_else( || { @@ -241,8 +240,16 @@ impl Data { }) } + #[inline] pub fn remove_keypair(&self) -> Result<()> { self.global.remove(b"keypair") } + /// TODO: the key valid until timestamp (`valid_until_ts`) is only honored + /// in room version > 4 + /// + /// Remove the outdated keys and insert the new ones. + /// + /// This doesn't actually check that the keys provided are newer than the + /// old set. pub fn add_signing_key( &self, origin: &ServerName, new_keys: ServerSigningKeys, ) -> Result> { @@ -306,14 +313,18 @@ impl Data { }) } + #[inline] pub fn bump_database_version(&self, new_version: u64) -> Result<()> { self.global.insert(b"version", &new_version.to_be_bytes())?; Ok(()) } + #[inline] pub fn backup(&self) -> Result<(), Box> { self.db.db.backup() } + #[inline] pub fn backup_list(&self) -> Result { self.db.db.backup_list() } + #[inline] pub fn file_list(&self) -> Result { self.db.db.file_list() } } diff --git a/src/service/globals/migrations.rs b/src/service/globals/migrations.rs index 5cc25a1d..e171cb9d 100644 --- a/src/service/globals/migrations.rs +++ b/src/service/globals/migrations.rs @@ -50,7 +50,10 @@ pub(crate) async fn migrations(db: &Arc, config: &Config) -> Result<() } async fn fresh(db: &Arc, config: &Config) -> Result<()> { - services().globals.bump_database_version(DATABASE_VERSION)?; + services() + .globals + .db + .bump_database_version(DATABASE_VERSION)?; db["global"].insert(b"fix_bad_double_separator_in_state_cache", &[])?; db["global"].insert(b"retroactively_fix_bad_data_from_roomuserid_joined", &[])?; @@ -68,57 +71,57 @@ async fn fresh(db: &Arc, config: &Config) -> Result<()> { /// Apply any migrations async fn migrate(db: &Arc, config: &Config) -> Result<()> { - if services().globals.database_version()? < 1 { + if services().globals.db.database_version()? < 1 { db_lt_1(db, config).await?; } - if services().globals.database_version()? < 2 { + if services().globals.db.database_version()? < 2 { db_lt_2(db, config).await?; } - if services().globals.database_version()? < 3 { + if services().globals.db.database_version()? < 3 { db_lt_3(db, config).await?; } - if services().globals.database_version()? < 4 { + if services().globals.db.database_version()? < 4 { db_lt_4(db, config).await?; } - if services().globals.database_version()? < 5 { + if services().globals.db.database_version()? < 5 { db_lt_5(db, config).await?; } - if services().globals.database_version()? < 6 { + if services().globals.db.database_version()? < 6 { db_lt_6(db, config).await?; } - if services().globals.database_version()? < 7 { + if services().globals.db.database_version()? < 7 { db_lt_7(db, config).await?; } - if services().globals.database_version()? < 8 { + if services().globals.db.database_version()? < 8 { db_lt_8(db, config).await?; } - if services().globals.database_version()? < 9 { + if services().globals.db.database_version()? < 9 { db_lt_9(db, config).await?; } - if services().globals.database_version()? < 10 { + if services().globals.db.database_version()? < 10 { db_lt_10(db, config).await?; } - if services().globals.database_version()? < 11 { + if services().globals.db.database_version()? < 11 { db_lt_11(db, config).await?; } - if services().globals.database_version()? < 12 { + if services().globals.db.database_version()? < 12 { db_lt_12(db, config).await?; } // This migration can be reused as-is anytime the server-default rules are // updated. - if services().globals.database_version()? < 13 { + if services().globals.db.database_version()? < 13 { db_lt_13(db, config).await?; } @@ -143,10 +146,10 @@ async fn migrate(db: &Arc, config: &Config) -> Result<()> { } assert_eq!( - services().globals.database_version().unwrap(), + services().globals.db.database_version().unwrap(), DATABASE_VERSION, "Failed asserting local database version {} is equal to known latest conduwuit database version {}", - services().globals.database_version().unwrap(), + services().globals.db.database_version().unwrap(), DATABASE_VERSION, ); @@ -225,7 +228,7 @@ async fn db_lt_1(db: &Arc, _config: &Config) -> Result<()> { serverroomids.insert(&serverroomid, &[])?; } - services().globals.bump_database_version(1)?; + services().globals.db.bump_database_version(1)?; info!("Migration: 0 -> 1 finished"); Ok(()) } @@ -242,7 +245,7 @@ async fn db_lt_2(db: &Arc, _config: &Config) -> Result<()> { } } - services().globals.bump_database_version(2)?; + services().globals.db.bump_database_version(2)?; info!("Migration: 1 -> 2 finished"); Ok(()) } @@ -262,7 +265,7 @@ async fn db_lt_3(db: &Arc, _config: &Config) -> Result<()> { mediaid_file.insert(&key, &[])?; } - services().globals.bump_database_version(3)?; + services().globals.db.bump_database_version(3)?; info!("Migration: 2 -> 3 finished"); Ok(()) } @@ -285,7 +288,7 @@ async fn db_lt_4(_db: &Arc, config: &Config) -> Result<()> { } } - services().globals.bump_database_version(4)?; + services().globals.db.bump_database_version(4)?; info!("Migration: 3 -> 4 finished"); Ok(()) } @@ -309,7 +312,7 @@ async fn db_lt_5(db: &Arc, _config: &Config) -> Result<()> { roomusertype_roomuserdataid.insert(&key, &roomuserdataid)?; } - services().globals.bump_database_version(5)?; + services().globals.db.bump_database_version(5)?; info!("Migration: 4 -> 5 finished"); Ok(()) } @@ -323,7 +326,7 @@ async fn db_lt_6(db: &Arc, _config: &Config) -> Result<()> { services().rooms.state_cache.update_joined_count(room_id)?; } - services().globals.bump_database_version(6)?; + services().globals.db.bump_database_version(6)?; info!("Migration: 5 -> 6 finished"); Ok(()) } @@ -448,7 +451,7 @@ async fn db_lt_7(db: &Arc, _config: &Config) -> Result<()> { )?; } - services().globals.bump_database_version(7)?; + services().globals.db.bump_database_version(7)?; info!("Migration: 6 -> 7 finished"); Ok(()) } @@ -514,7 +517,7 @@ async fn db_lt_8(db: &Arc, _config: &Config) -> Result<()> { eventid_pduid.insert_batch(batch2.iter().map(database::KeyVal::from))?; - services().globals.bump_database_version(8)?; + services().globals.db.bump_database_version(8)?; info!("Migration: 7 -> 8 finished"); Ok(()) } @@ -571,7 +574,7 @@ async fn db_lt_9(db: &Arc, _config: &Config) -> Result<()> { tokenids.remove(&key)?; } - services().globals.bump_database_version(9)?; + services().globals.db.bump_database_version(9)?; info!("Migration: 8 -> 9 finished"); Ok(()) } @@ -590,7 +593,7 @@ async fn db_lt_10(db: &Arc, _config: &Config) -> Result<()> { services().users.mark_device_key_update(&user_id)?; } - services().globals.bump_database_version(10)?; + services().globals.db.bump_database_version(10)?; info!("Migration: 9 -> 10 finished"); Ok(()) } @@ -601,7 +604,7 @@ async fn db_lt_11(_db: &Arc, _config: &Config) -> Result<()> { //let userdevicesessionid_uiaarequest = &db["userdevicesessionid_uiaarequest"]; //userdevicesessionid_uiaarequest.clear()?; - services().globals.bump_database_version(11)?; + services().globals.db.bump_database_version(11)?; info!("Migration: 10 -> 11 finished"); Ok(()) } @@ -669,7 +672,7 @@ async fn db_lt_12(_db: &Arc, config: &Config) -> Result<()> { )?; } - services().globals.bump_database_version(12)?; + services().globals.db.bump_database_version(12)?; info!("Migration: 11 -> 12 finished"); Ok(()) } @@ -706,7 +709,7 @@ async fn db_lt_13(_db: &Arc, config: &Config) -> Result<()> { )?; } - services().globals.bump_database_version(13)?; + services().globals.db.bump_database_version(13)?; info!("Migration: 12 -> 13 finished"); Ok(()) } @@ -736,8 +739,8 @@ async fn migrate_sha256_media(db: &Arc, _config: &Config) -> Result<() // Apply fix from when sha256_media was backward-incompat and bumped the schema // version from 13 to 14. For users satisfying these conditions we can go back. - if services().globals.database_version()? == 14 && DATABASE_VERSION == 13 { - services().globals.bump_database_version(13)?; + if services().globals.db.database_version()? == 14 && DATABASE_VERSION == 13 { + services().globals.db.bump_database_version(13)?; } db["global"].insert(b"feat_sha256_media", &[])?; diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 2a6fbc2f..12843b9d 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -17,10 +17,7 @@ use data::Data; use ipaddress::IPAddress; use regex::RegexSet; use ruma::{ - api::{ - client::discovery::discover_support::ContactRole, - federation::discovery::{ServerSigningKeys, VerifyKey}, - }, + api::{client::discovery::discover_support::ContactRole, federation::discovery::VerifyKey}, serde::Base64, DeviceId, OwnedEventId, OwnedRoomAliasId, OwnedRoomId, OwnedServerName, OwnedServerSigningKeyId, OwnedUserId, RoomAliasId, RoomVersionId, ServerName, UserId, @@ -230,6 +227,7 @@ impl Service { pub fn allow_unstable_room_versions(&self) -> bool { self.config.allow_unstable_room_versions } + #[inline] pub fn default_room_version(&self) -> RoomVersionId { self.config.default_room_version.clone() } pub fn new_user_displayname_suffix(&self) -> &String { &self.config.new_user_displayname_suffix } @@ -317,19 +315,6 @@ impl Service { room_versions } - /// TODO: the key valid until timestamp (`valid_until_ts`) is only honored - /// in room version > 4 - /// - /// Remove the outdated keys and insert the new ones. - /// - /// This doesn't actually check that the keys provided are newer than the - /// old set. - pub fn add_signing_key( - &self, origin: &ServerName, new_keys: ServerSigningKeys, - ) -> Result> { - self.db.add_signing_key(origin, new_keys) - } - /// This returns an empty `Ok(BTreeMap<..>)` when there are no keys found /// for the server. pub fn signing_keys_for(&self, origin: &ServerName) -> Result> { @@ -348,14 +333,11 @@ impl Service { Ok(keys) } - pub fn database_version(&self) -> Result { self.db.database_version() } - - pub fn bump_database_version(&self, new_version: u64) -> Result<()> { self.db.bump_database_version(new_version) } - pub fn well_known_client(&self) -> &Option { &self.config.well_known.client } pub fn well_known_server(&self) -> &Option { &self.config.well_known.server } + #[inline] pub fn valid_cidr_range(&self, ip: &IPAddress) -> bool { for cidr in &self.cidr_range_denylist { if cidr.includes(ip) { diff --git a/src/service/media/data.rs b/src/service/media/data.rs index 4cabf167..186d502d 100644 --- a/src/service/media/data.rs +++ b/src/service/media/data.rs @@ -148,6 +148,7 @@ impl Data { /// associated with it such as width, height, content-type, etc) pub(crate) fn get_all_media_keys(&self) -> Vec> { self.mediaid_file.iter().map(|(key, _)| key).collect() } + #[inline] pub(super) fn remove_url_preview(&self, url: &str) -> Result<()> { self.url_previews.remove(url.as_bytes()) } pub(super) fn set_url_preview( diff --git a/src/service/presence/mod.rs b/src/service/presence/mod.rs index 6f05d402..584f1a6d 100644 --- a/src/service/presence/mod.rs +++ b/src/service/presence/mod.rs @@ -131,6 +131,7 @@ impl crate::Service for Service { impl Service { /// Returns the latest presence event for the given user. + #[inline] pub fn get_presence(&self, user_id: &UserId) -> Result> { if let Some((_, presence)) = self.db.get_presence(user_id)? { Ok(Some(presence)) @@ -207,6 +208,7 @@ impl Service { /// Returns the most recent presence updates that happened after the event /// with id `since`. + #[inline] pub fn presence_since(&self, since: u64) -> Box)> + '_> { self.db.presence_since(since) } diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index bb22c6c8..9f50ef58 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -1381,6 +1381,7 @@ impl Service { Ok(create_event_content.room_version) } + #[inline] fn to_room_version(room_version_id: &RoomVersionId) -> RoomVersion { RoomVersion::new(room_version_id).expect("room version is supported") } diff --git a/src/service/rooms/event_handler/signing_keys.rs b/src/service/rooms/event_handler/signing_keys.rs index d22af9bd..fdb2be0e 100644 --- a/src/service/rooms/event_handler/signing_keys.rs +++ b/src/service/rooms/event_handler/signing_keys.rs @@ -201,6 +201,7 @@ impl super::Service { let result = services() .globals + .db .add_signing_key(&k.server_name, k.clone())? .into_iter() .map(|(k, v)| (k.to_string(), v.key)) @@ -249,6 +250,7 @@ impl super::Service { if let Ok(key) = get_keys_response.server_key.deserialize() { let result: BTreeMap<_, _> = services() .globals + .db .add_signing_key(&origin, key)? .into_iter() .map(|(k, v)| (k.to_string(), v.key)) @@ -392,7 +394,7 @@ impl super::Service { }) { debug!("Got signing keys: {:?}", server_keys); for k in server_keys { - services().globals.add_signing_key(origin, k.clone())?; + services().globals.db.add_signing_key(origin, k.clone())?; result.extend( k.verify_keys .into_iter() @@ -421,6 +423,7 @@ impl super::Service { { services() .globals + .db .add_signing_key(origin, server_key.clone())?; result.extend( @@ -453,6 +456,7 @@ impl super::Service { { services() .globals + .db .add_signing_key(origin, server_key.clone())?; result.extend( @@ -499,7 +503,7 @@ impl super::Service { }) { debug!("Got signing keys: {:?}", server_keys); for k in server_keys { - services().globals.add_signing_key(origin, k.clone())?; + services().globals.db.add_signing_key(origin, k.clone())?; result.extend( k.verify_keys .into_iter() diff --git a/src/service/rooms/metadata/data.rs b/src/service/rooms/metadata/data.rs index df9b6b10..763dd0e8 100644 --- a/src/service/rooms/metadata/data.rs +++ b/src/service/rooms/metadata/data.rs @@ -48,10 +48,12 @@ impl Data { })) } + #[inline] pub(super) fn is_disabled(&self, room_id: &RoomId) -> Result { Ok(self.disabledroomids.get(room_id.as_bytes())?.is_some()) } + #[inline] pub(super) fn disable_room(&self, room_id: &RoomId, disabled: bool) -> Result<()> { if disabled { self.disabledroomids.insert(room_id.as_bytes(), &[])?; @@ -62,10 +64,12 @@ impl Data { Ok(()) } + #[inline] pub(super) fn is_banned(&self, room_id: &RoomId) -> Result { Ok(self.bannedroomids.get(room_id.as_bytes())?.is_some()) } + #[inline] pub(super) fn ban_room(&self, room_id: &RoomId, banned: bool) -> Result<()> { if banned { self.bannedroomids.insert(room_id.as_bytes(), &[])?; diff --git a/src/service/rooms/metadata/mod.rs b/src/service/rooms/metadata/mod.rs index 3a96b89f..ec34a82c 100644 --- a/src/service/rooms/metadata/mod.rs +++ b/src/service/rooms/metadata/mod.rs @@ -22,22 +22,27 @@ impl crate::Service for Service { impl Service { /// Checks if a room exists. - #[tracing::instrument(skip(self))] + #[inline] pub fn exists(&self, room_id: &RoomId) -> Result { self.db.exists(room_id) } #[must_use] pub fn iter_ids<'a>(&'a self) -> Box> + 'a> { self.db.iter_ids() } + #[inline] pub fn is_disabled(&self, room_id: &RoomId) -> Result { self.db.is_disabled(room_id) } + #[inline] pub fn disable_room(&self, room_id: &RoomId, disabled: bool) -> Result<()> { self.db.disable_room(room_id, disabled) } + #[inline] pub fn is_banned(&self, room_id: &RoomId) -> Result { self.db.is_banned(room_id) } + #[inline] pub fn ban_room(&self, room_id: &RoomId, banned: bool) -> Result<()> { self.db.ban_room(room_id, banned) } + #[inline] #[must_use] pub fn list_banned_rooms<'a>(&'a self) -> Box> + 'a> { self.db.list_banned_rooms() diff --git a/src/service/rooms/state/data.rs b/src/service/rooms/state/data.rs index dad613d2..aad3bede 100644 --- a/src/service/rooms/state/data.rs +++ b/src/service/rooms/state/data.rs @@ -30,6 +30,7 @@ impl Data { }) } + #[inline] pub(super) fn set_room_state( &self, room_id: &RoomId, diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index bf3b29b7..7964a5fb 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -200,6 +200,7 @@ impl Service { if let Some(state_key) = &new_pdu.state_key { let states_parents = previous_shortstatehash.map_or_else( || Ok(Vec::new()), + #[inline] |p| { services() .rooms @@ -344,6 +345,7 @@ impl Service { Ok(create_event_content.room_version) } + #[inline] pub fn get_room_shortstatehash(&self, room_id: &RoomId) -> Result> { self.db.get_room_shortstatehash(room_id) } diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index ac127b35..7fe61134 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -96,6 +96,7 @@ impl Service { /// Returns a single PDU from `room_id` with key (`event_type`, /// `state_key`). + #[inline] pub fn state_get( &self, shortstatehash: u64, event_type: &StateEventType, state_key: &str, ) -> Result>> { @@ -113,6 +114,7 @@ impl Service { } /// The user was a joined member at this state (potentially in the past) + #[inline] fn user_was_joined(&self, shortstatehash: u64, user_id: &UserId) -> bool { self.user_membership(shortstatehash, user_id) .is_ok_and(|s| s == MembershipState::Join) @@ -122,6 +124,7 @@ impl Service { /// The user was an invited or joined room member at this state (potentially /// in the past) + #[inline] fn user_was_invited(&self, shortstatehash: u64, user_id: &UserId) -> bool { self.user_membership(shortstatehash, user_id) .is_ok_and(|s| s == MembershipState::Join || s == MembershipState::Invite) diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index 564a0bee..97f3fb80 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -135,6 +135,7 @@ impl Service { } /// Returns shortstatekey, event id + #[inline] pub fn parse_compressed_state_event(&self, compressed_event: &CompressedStateEvent) -> Result<(u64, Arc)> { Ok(( utils::u64_from_bytes(&compressed_event[0..size_of::()]).expect("bytes have right length"), diff --git a/src/service/rooms/timeline/data.rs b/src/service/rooms/timeline/data.rs index 2054caf7..0d4d945e 100644 --- a/src/service/rooms/timeline/data.rs +++ b/src/service/rooms/timeline/data.rs @@ -97,6 +97,7 @@ impl Data { } /// Returns the pdu's id. + #[inline] pub(super) fn get_pdu_id(&self, event_id: &EventId) -> Result>> { self.eventid_pduid.get(event_id.as_bytes()) } diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index f7e8384b..ba987dbd 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -175,11 +175,13 @@ impl Service { } /// Returns the json of a pdu. + #[inline] pub fn get_non_outlier_pdu_json(&self, event_id: &EventId) -> Result> { self.db.get_non_outlier_pdu_json(event_id) } /// Returns the pdu's id. + #[inline] pub fn get_pdu_id(&self, event_id: &EventId) -> Result>> { self.db.get_pdu_id(event_id) } @@ -190,6 +192,7 @@ impl Service { /// /// TODO: use this? #[allow(dead_code)] + #[inline] pub fn get_non_outlier_pdu(&self, event_id: &EventId) -> Result> { self.db.get_non_outlier_pdu(event_id) } @@ -1017,6 +1020,7 @@ impl Service { } /// Returns an iterator over all PDUs in a room. + #[inline] pub fn all_pdus<'a>( &'a self, user_id: &UserId, room_id: &RoomId, ) -> Result> + 'a> { diff --git a/src/service/sending/data.rs b/src/service/sending/data.rs index 22b69818..ed579817 100644 --- a/src/service/sending/data.rs +++ b/src/service/sending/data.rs @@ -27,6 +27,7 @@ impl Data { } } + #[inline] pub fn active_requests(&self) -> OutgoingSendingIter<'_> { Box::new( self.servercurrentevent_data @@ -35,6 +36,7 @@ impl Data { ) } + #[inline] pub fn active_requests_for<'a>(&'a self, destination: &Destination) -> SendingEventIter<'a> { let prefix = destination.get_prefix(); Box::new( diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index b75ca8b4..d3f3cbdd 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -1,6 +1,6 @@ mod appservice; mod data; -pub mod resolve; +mod resolve; mod send; mod sender; @@ -9,7 +9,7 @@ use std::{fmt::Debug, sync::Arc}; use async_trait::async_trait; use conduit::{Error, Result}; use data::Data; -pub use resolve::FedDest; +pub use resolve::{resolve_actual_dest, FedDest}; use ruma::{ api::{appservice::Registration, OutgoingRequest}, OwnedServerName, OwnedUserId, RoomId, ServerName, UserId, diff --git a/src/service/sending/resolve.rs b/src/service/sending/resolve.rs index 51e7a92f..5f257217 100644 --- a/src/service/sending/resolve.rs +++ b/src/service/sending/resolve.rs @@ -414,6 +414,7 @@ impl FedDest { } } + #[inline] fn port(&self) -> Option { match &self { Self::Literal(addr) => Some(addr.port()), diff --git a/src/service/users/data.rs b/src/service/users/data.rs index 14aa8006..3ba54a93 100644 --- a/src/service/users/data.rs +++ b/src/service/users/data.rs @@ -61,6 +61,7 @@ impl Data { } /// Check if a user has an account on this homeserver. + #[inline] pub(super) fn exists(&self, user_id: &UserId) -> Result { Ok(self.userid_password.get(user_id.as_bytes())?.is_some()) } @@ -75,6 +76,7 @@ impl Data { } /// Returns the number of users registered on this server. + #[inline] pub(super) fn count(&self) -> Result { Ok(self.userid_password.iter().count()) } /// Find out which user an access token belongs to. diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index cd580994..0196e1aa 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -53,6 +53,7 @@ impl crate::Service for Service { impl Service { /// Check if a user has an account on this homeserver. + #[inline] pub fn exists(&self, user_id: &UserId) -> Result { self.db.exists(user_id) } pub fn forget_sync_request_connection(&self, user_id: OwnedUserId, device_id: OwnedDeviceId, conn_id: String) { @@ -257,12 +258,14 @@ impl Service { } /// Create a new user account on this homeserver. + #[inline] pub fn create(&self, user_id: &UserId, password: Option<&str>) -> Result<()> { self.db.set_password(user_id, password)?; Ok(()) } /// Returns the number of users registered on this server. + #[inline] pub fn count(&self) -> Result { self.db.count() } /// Find out which user an access token belongs to. @@ -283,6 +286,7 @@ impl Service { pub fn password_hash(&self, user_id: &UserId) -> Result> { self.db.password_hash(user_id) } /// Hash and set the user's password to the Argon2 hash + #[inline] pub fn set_password(&self, user_id: &UserId, password: Option<&str>) -> Result<()> { self.db.set_password(user_id, password) } @@ -331,6 +335,7 @@ impl Service { } /// Replaces the access token of one device. + #[inline] pub fn set_token(&self, user_id: &UserId, device_id: &DeviceId, token: &str) -> Result<()> { self.db.set_token(user_id, device_id, token) } @@ -385,18 +390,21 @@ impl Service { self.db.keys_changed(user_or_room_id, from, to) } + #[inline] pub fn mark_device_key_update(&self, user_id: &UserId) -> Result<()> { self.db.mark_device_key_update(user_id) } pub fn get_device_keys(&self, user_id: &UserId, device_id: &DeviceId) -> Result>> { self.db.get_device_keys(user_id, device_id) } + #[inline] pub fn parse_master_key( &self, user_id: &UserId, master_key: &Raw, ) -> Result<(Vec, CrossSigningKey)> { Data::parse_master_key(user_id, master_key) } + #[inline] pub fn get_key( &self, key: &[u8], sender_user: Option<&UserId>, user_id: &UserId, allowed_signatures: &dyn Fn(&UserId) -> bool, ) -> Result>> { From eaf1cf38a52caa6e82d5beb9539fa35d8b818a3e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 4 Jul 2024 11:44:39 +0000 Subject: [PATCH 011/158] refactor resolver tuples into structs Signed-off-by: Jason Volk --- src/service/globals/mod.rs | 2 +- src/service/globals/resolver.rs | 22 ++++---- src/service/sending/mod.rs | 2 +- src/service/sending/resolve.rs | 93 ++++++++++++++++++++++++--------- src/service/sending/send.rs | 21 ++++---- 5 files changed, 93 insertions(+), 47 deletions(-) diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 12843b9d..3c0e66c0 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -2,7 +2,7 @@ mod client; mod data; pub(super) mod emerg_access; pub(super) mod migrations; -mod resolver; +pub(crate) mod resolver; pub(super) mod updates; use std::{ diff --git a/src/service/globals/resolver.rs b/src/service/globals/resolver.rs index e5199162..0cfaa1b9 100644 --- a/src/service/globals/resolver.rs +++ b/src/service/globals/resolver.rs @@ -12,21 +12,21 @@ use hickory_resolver::TokioAsyncResolver; use reqwest::dns::{Addrs, Name, Resolve, Resolving}; use ruma::OwnedServerName; -use crate::sending::FedDest; +use crate::sending::{CachedDest, CachedOverride}; -type WellKnownMap = HashMap; -type TlsNameMap = HashMap, u16)>; +type WellKnownMap = HashMap; +type TlsNameMap = HashMap; pub struct Resolver { pub destinations: Arc>, // actual_destination, host pub overrides: Arc>, - pub resolver: Arc, - pub hooked: Arc, + pub(crate) resolver: Arc, + pub(crate) hooked: Arc, } -pub struct Hooked { - pub overrides: Arc>, - pub resolver: Arc, +pub(crate) struct Hooked { + overrides: Arc>, + resolver: Arc, } impl Resolver { @@ -117,15 +117,15 @@ impl Resolve for Resolver { impl Resolve for Hooked { fn resolve(&self, name: Name) -> Resolving { - let addr_port = self + let cached = self .overrides .read() .expect("locked for reading") .get(name.as_str()) .cloned(); - if let Some((addr, port)) = addr_port { - cached_to_reqwest(&addr, port) + if let Some(cached) = cached { + cached_to_reqwest(&cached.ips, cached.port) } else { resolve_to_reqwest(self.resolver.clone(), name) } diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index d3f3cbdd..0dcebcec 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -9,7 +9,7 @@ use std::{fmt::Debug, sync::Arc}; use async_trait::async_trait; use conduit::{Error, Result}; use data::Data; -pub use resolve::{resolve_actual_dest, FedDest}; +pub use resolve::{resolve_actual_dest, CachedDest, CachedOverride, FedDest}; use ruma::{ api::{appservice::Registration, OutgoingRequest}, OwnedServerName, OwnedUserId, RoomId, ServerName, UserId, diff --git a/src/service/sending/resolve.rs b/src/service/sending/resolve.rs index 5f257217..01943cc0 100644 --- a/src/service/sending/resolve.rs +++ b/src/service/sending/resolve.rs @@ -6,7 +6,7 @@ use std::{ use hickory_resolver::{error::ResolveError, lookup::SrvLookup}; use ipaddress::IPAddress; -use ruma::ServerName; +use ruma::{OwnedServerName, ServerName}; use tracing::{debug, error, trace}; use crate::{debug_error, debug_info, debug_warn, services, Error, Result}; @@ -35,6 +35,7 @@ pub enum FedDest { Named(String, String), } +#[derive(Clone, Debug)] pub(crate) struct ActualDest { pub(crate) dest: FedDest, pub(crate) host: String, @@ -42,19 +43,31 @@ pub(crate) struct ActualDest { pub(crate) cached: bool, } +#[derive(Clone, Debug)] +pub struct CachedDest { + pub dest: FedDest, + pub host: String, +} + +#[derive(Clone, Debug)] +pub struct CachedOverride { + pub ips: Vec, + pub port: u16, +} + #[tracing::instrument(skip_all, name = "resolve")] pub(crate) async fn get_actual_dest(server_name: &ServerName) -> Result { let cached; let cached_result = services() .globals .resolver - .destinations - .read() - .expect("locked for reading") - .get(server_name) - .cloned(); + .get_cached_destination(server_name); - let (dest, host) = if let Some(result) = cached_result { + let CachedDest { + dest, + host, + .. + } = if let Some(result) = cached_result { cached = true; result } else { @@ -77,7 +90,7 @@ pub(crate) async fn get_actual_dest(server_name: &ServerName) -> Result Result<(FedDest, String)> { +pub async fn resolve_actual_dest(dest: &ServerName, cache: bool) -> Result { trace!("Finding actual destination for {dest}"); let mut host = dest.as_str().to_owned(); let actual_dest = match get_ip_with_port(dest.as_str()) { @@ -109,7 +122,10 @@ pub async fn resolve_actual_dest(dest: &ServerName, cache: bool) -> Result<(FedD }; debug!("Actual destination: {actual_dest:?} hostname: {host:?}"); - Ok((actual_dest, host.into_uri_string())) + Ok(CachedDest { + dest: actual_dest, + host: host.into_uri_string(), + }) } fn actual_dest_1(host_port: FedDest) -> Result { @@ -193,14 +209,7 @@ async fn actual_dest_5(dest: &ServerName, cache: bool) -> Result { #[tracing::instrument(skip_all, name = "well-known")] async fn request_well_known(dest: &str) -> Result> { trace!("Requesting well known for {dest}"); - if !services() - .globals - .resolver - .overrides - .read() - .unwrap() - .contains_key(dest) - { + if !services().globals.resolver.has_cached_override(dest) { query_and_cache_override(dest, dest, 8448).await?; } @@ -269,15 +278,16 @@ async fn query_and_cache_override(overname: &'_ str, hostname: &'_ str, port: u1 Err(e) => handle_resolve_error(&e), Ok(override_ip) => { if hostname != overname { - debug_info!("{:?} overriden by {:?}", overname, hostname); + debug_info!("{overname:?} overriden by {hostname:?}"); } - services() - .globals - .resolver - .overrides - .write() - .unwrap() - .insert(overname.to_owned(), (override_ip.iter().collect(), port)); + + services().globals.resolver.set_cached_override( + overname.to_owned(), + CachedOverride { + ips: override_ip.iter().collect(), + port, + }, + ); Ok(()) }, @@ -392,6 +402,39 @@ fn add_port_to_hostname(dest_str: &str) -> FedDest { FedDest::Named(host.to_owned(), port.to_owned()) } +impl crate::globals::resolver::Resolver { + pub(crate) fn set_cached_destination(&self, name: OwnedServerName, dest: CachedDest) -> Option { + trace!(?name, ?dest, "set cached destination"); + self.destinations + .write() + .expect("locked for writing") + .insert(name, dest) + } + + pub(crate) fn get_cached_destination(&self, name: &ServerName) -> Option { + self.destinations + .read() + .expect("locked for reading") + .get(name) + .cloned() + } + + pub(crate) fn set_cached_override(&self, name: String, over: CachedOverride) -> Option { + trace!(?name, ?over, "set cached override"); + self.overrides + .write() + .expect("locked for writing") + .insert(name, over) + } + + pub(crate) fn has_cached_override(&self, name: &str) -> bool { + self.overrides + .read() + .expect("locked for reading") + .contains_key(name) + } +} + impl FedDest { fn into_https_string(self) -> String { match self { diff --git a/src/service/sending/send.rs b/src/service/sending/send.rs index 57ff9127..1b977a73 100644 --- a/src/service/sending/send.rs +++ b/src/service/sending/send.rs @@ -8,11 +8,14 @@ use ruma::{ client::error::Error as RumaError, EndpointError, IncomingResponse, MatrixVersion, OutgoingRequest, SendAccessToken, }, - OwnedServerName, ServerName, + ServerName, }; use tracing::{debug, trace}; -use super::{resolve, resolve::ActualDest}; +use super::{ + resolve, + resolve::{ActualDest, CachedDest}, +}; use crate::{debug_error, debug_warn, services, Error, Result}; #[tracing::instrument(skip_all, name = "send")] @@ -103,13 +106,13 @@ where let response = T::IncomingResponse::try_from_http_response(http_response); if response.is_ok() && !actual.cached { - services() - .globals - .resolver - .destinations - .write() - .expect("locked for writing") - .insert(OwnedServerName::from(dest), (actual.dest.clone(), actual.host.clone())); + services().globals.resolver.set_cached_destination( + dest.to_owned(), + CachedDest { + dest: actual.dest.clone(), + host: actual.host.clone(), + }, + ); } match response { From dc18f89c0bd851aa861b59cdf78342176b31f426 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 4 Jul 2024 11:46:30 +0000 Subject: [PATCH 012/158] don't cache server name lookups indefinitely (#436) Signed-off-by: Jason Volk --- src/service/globals/resolver.rs | 1 + src/service/sending/resolve.rs | 30 ++++++++++++++++++++++++++++-- src/service/sending/send.rs | 1 + 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/service/globals/resolver.rs b/src/service/globals/resolver.rs index 0cfaa1b9..3082f2fd 100644 --- a/src/service/globals/resolver.rs +++ b/src/service/globals/resolver.rs @@ -122,6 +122,7 @@ impl Resolve for Hooked { .read() .expect("locked for reading") .get(name.as_str()) + .filter(|cached| cached.valid()) .cloned(); if let Some(cached) = cached { diff --git a/src/service/sending/resolve.rs b/src/service/sending/resolve.rs index 01943cc0..77311006 100644 --- a/src/service/sending/resolve.rs +++ b/src/service/sending/resolve.rs @@ -2,6 +2,7 @@ use std::{ fmt, fmt::Debug, net::{IpAddr, SocketAddr}, + time::SystemTime, }; use hickory_resolver::{error::ResolveError, lookup::SrvLookup}; @@ -9,7 +10,7 @@ use ipaddress::IPAddress; use ruma::{OwnedServerName, ServerName}; use tracing::{debug, error, trace}; -use crate::{debug_error, debug_info, debug_warn, services, Error, Result}; +use crate::{debug_error, debug_info, debug_warn, services, utils::rand, Error, Result}; /// Wraps either an literal IP address plus port, or a hostname plus complement /// (colon-plus-port if it was specified). @@ -47,12 +48,14 @@ pub(crate) struct ActualDest { pub struct CachedDest { pub dest: FedDest, pub host: String, + pub expire: SystemTime, } #[derive(Clone, Debug)] pub struct CachedOverride { pub ips: Vec, pub port: u16, + pub expire: SystemTime, } #[tracing::instrument(skip_all, name = "resolve")] @@ -125,6 +128,7 @@ pub async fn resolve_actual_dest(dest: &ServerName, cache: bool) -> Result bool { self.expire > SystemTime::now() } + + #[must_use] + pub(crate) fn default_expire() -> SystemTime { rand::timepoint_secs(60 * 60 * 18..60 * 60 * 36) } +} + +impl CachedOverride { + #[inline] + #[must_use] + pub fn valid(&self) -> bool { self.expire > SystemTime::now() } + + #[must_use] + pub(crate) fn default_expire() -> SystemTime { rand::timepoint_secs(60 * 60 * 6..60 * 60 * 12) } +} + impl FedDest { fn into_https_string(self) -> String { match self { diff --git a/src/service/sending/send.rs b/src/service/sending/send.rs index 1b977a73..a89ea2f8 100644 --- a/src/service/sending/send.rs +++ b/src/service/sending/send.rs @@ -111,6 +111,7 @@ where CachedDest { dest: actual.dest.clone(), host: actual.host.clone(), + expire: CachedDest::default_expire(), }, ); } From 2dd68d3fa5fa48cb19864aa0f35bf825cf28d96e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 4 Jul 2024 12:58:53 +0000 Subject: [PATCH 013/158] add time format string util Signed-off-by: Jason Volk --- Cargo.toml | 2 +- src/core/utils/time.rs | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a52b4138..44f43849 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -208,7 +208,7 @@ features = ["serde"] # standard date and time tools [workspace.dependencies.chrono] version = "0.4.38" -features = ["alloc"] +features = ["alloc", "std"] default-features = false [workspace.dependencies.hyper] diff --git a/src/core/utils/time.rs b/src/core/utils/time.rs index f6208b31..7de00e9e 100644 --- a/src/core/utils/time.rs +++ b/src/core/utils/time.rs @@ -1,9 +1,9 @@ +use std::time::{SystemTime, UNIX_EPOCH}; + #[inline] #[must_use] #[allow(clippy::as_conversions)] pub fn now_millis() -> u64 { - use std::time::UNIX_EPOCH; - UNIX_EPOCH .elapsed() .expect("positive duration after epoch") @@ -18,3 +18,11 @@ pub fn rfc2822_from_seconds(epoch: i64) -> String { .unwrap_or_default() .to_rfc2822() } + +#[must_use] +pub fn format(ts: SystemTime, str: &str) -> String { + use chrono::{DateTime, Utc}; + + let dt: DateTime = ts.into(); + dt.format(str).to_string() +} From 17a3ed4c560c0890a20776769ea1758acb6b1920 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 4 Jul 2024 11:49:13 +0000 Subject: [PATCH 014/158] add admin query resolver commands Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 15 ++++--- src/admin/query/mod.rs | 27 ++++++++++-- src/admin/query/resolver.rs | 87 +++++++++++++++++++++++++++++++++++++ 3 files changed, 119 insertions(+), 10 deletions(-) create mode 100644 src/admin/query/resolver.rs diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 23197431..3efe283b 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -614,15 +614,16 @@ pub(super) async fn resolve_true_destination( let state = &services().server.log.capture; let logs = Arc::new(Mutex::new(String::new())); let capture = Capture::new(state, Some(filter), capture::fmt_markdown(logs.clone())); - let (actual_dest, hostname_uri); - { - let _capture_scope = capture.start(); - (actual_dest, hostname_uri) = resolve_actual_dest(&server_name, !no_cache).await?; - }; + + let capture_scope = capture.start(); + let actual = resolve_actual_dest(&server_name, !no_cache).await?; + drop(capture_scope); let msg = format!( - "{}\nDestination: {actual_dest}\nHostname URI: {hostname_uri}", - logs.lock().expect("locked") + "{}\nDestination: {}\nHostname URI: {}", + logs.lock().expect("locked"), + actual.dest, + actual.host, ); Ok(RoomMessageEventContent::text_markdown(msg)) } diff --git a/src/admin/query/mod.rs b/src/admin/query/mod.rs index 946e6ec8..ea7036d0 100644 --- a/src/admin/query/mod.rs +++ b/src/admin/query/mod.rs @@ -2,6 +2,7 @@ mod account_data; mod appservice; mod globals; mod presence; +mod resolver; mod room_alias; mod room_state_cache; mod sending; @@ -12,12 +13,12 @@ use conduit::Result; use room_state_cache::room_state_cache; use ruma::{ events::{room::message::RoomMessageEventContent, RoomAccountDataEventType}, - RoomAliasId, RoomId, ServerName, UserId, + OwnedServerName, RoomAliasId, RoomId, ServerName, UserId, }; use self::{ - account_data::account_data, appservice::appservice, globals::globals, presence::presence, room_alias::room_alias, - sending::sending, users::users, + account_data::account_data, appservice::appservice, globals::globals, presence::presence, resolver::resolver, + room_alias::room_alias, sending::sending, users::users, }; #[cfg_attr(test, derive(Debug))] @@ -55,6 +56,10 @@ pub(super) enum QueryCommand { /// - users.rs iterators and getters #[command(subcommand)] Users(Users), + + /// - resolver service + #[command(subcommand)] + Resolver(Resolver), } #[cfg_attr(test, derive(Debug))] @@ -287,6 +292,21 @@ pub(super) enum Users { Iter, } +#[cfg_attr(test, derive(Debug))] +#[derive(Subcommand)] +/// Resolver service and caches +pub(super) enum Resolver { + /// Query the destinations cache + DestinationsCache { + server_name: Option, + }, + + /// Query the overrides cache + OverridesCache { + name: Option, + }, +} + /// Processes admin query commands pub(super) async fn process(command: QueryCommand, _body: Vec<&str>) -> Result { Ok(match command { @@ -298,5 +318,6 @@ pub(super) async fn process(command: QueryCommand, _body: Vec<&str>) -> Result globals(command).await?, QueryCommand::Sending(command) => sending(command).await?, QueryCommand::Users(command) => users(command).await?, + QueryCommand::Resolver(command) => resolver(command).await?, }) } diff --git a/src/admin/query/resolver.rs b/src/admin/query/resolver.rs new file mode 100644 index 00000000..2a2554b5 --- /dev/null +++ b/src/admin/query/resolver.rs @@ -0,0 +1,87 @@ +use std::fmt::Write; + +use conduit::{utils::time, Result}; +use ruma::{events::room::message::RoomMessageEventContent, OwnedServerName}; + +use super::Resolver; +use crate::services; + +/// All the getters and iterators in key_value/users.rs +pub(super) async fn resolver(subcommand: Resolver) -> Result { + match subcommand { + Resolver::DestinationsCache { + server_name, + } => destinations_cache(server_name).await, + Resolver::OverridesCache { + name, + } => overrides_cache(name).await, + } +} + +async fn destinations_cache(server_name: Option) -> Result { + use service::sending::CachedDest; + + let mut out = String::new(); + writeln!(out, "| Server Name | Destination | Hostname | Expires |")?; + writeln!(out, "| ----------- | ----------- | -------- | ------- |")?; + let row = |( + name, + &CachedDest { + ref dest, + ref host, + expire, + }, + )| { + let expire = time::format(expire, "%+"); + writeln!(out, "| {name} | {dest} | {host} | {expire} |").expect("wrote line"); + }; + + let map = services() + .globals + .resolver + .destinations + .read() + .expect("locked"); + + if let Some(server_name) = server_name.as_ref() { + map.get_key_value(server_name).map(row); + } else { + map.iter().for_each(row); + } + + Ok(RoomMessageEventContent::notice_markdown(out)) +} + +async fn overrides_cache(server_name: Option) -> Result { + use service::sending::CachedOverride; + + let mut out = String::new(); + writeln!(out, "| Server Name | IP | Port | Expires |")?; + writeln!(out, "| ----------- | --- | ----:| ------- |")?; + let row = |( + name, + &CachedOverride { + ref ips, + port, + expire, + }, + )| { + let expire = time::format(expire, "%+"); + writeln!(out, "| {name} | {ips:?} | {port} | {expire} |").expect("wrote line"); + }; + + let map = services() + .globals + .resolver + .overrides + .read() + .expect("locked"); + + if let Some(server_name) = server_name.as_ref() { + map.get_key_value(server_name).map(row); + } else { + map.iter().for_each(row); + } + + Ok(RoomMessageEventContent::notice_markdown(out)) +} From 8691141237c459ffe66ec13083d22757c318d302 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 5 Jul 2024 02:48:36 -0400 Subject: [PATCH 015/158] ci: try running `nix-build-and-cache` prior dynamic build test Signed-off-by: strawberry --- engage.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/engage.toml b/engage.toml index 365c0179..d28ecff1 100644 --- a/engage.toml +++ b/engage.toml @@ -184,5 +184,6 @@ cargo test \ name = "nix-default" group = "tests" script = """ -nix run .#default -- --help +bin/nix-build-and-cache just .#default +nix run -L .#default -- --help """ From 2bc53139faed06653b05ab6ce3f94e9d89905fe5 Mon Sep 17 00:00:00 2001 From: Christoph Dittmann Date: Sat, 6 Jul 2024 14:08:14 +0200 Subject: [PATCH 016/158] Don't send empty presence EDUs I run a homeserver whose logs show a high number of incoming empty presence EDUs originating from the user agent "Conduwuit/0.4.4". They arrive at a rate of about 2 queries per second per Conduwuit server. The empty EDUs all look the same, only with `origin_server_ts` increasing: ``` {"origin":"example.com","origin_server_ts":1720266475601,"edus":[{"edu_type":"m.presence","content":{"push":[]}}]} ``` These updates are unnecessary because they don't do anything. They only increase network traffic and CPU usage on both sides. After this commit, the empty presence updates are no longer inserted into the outgoing event queue. --- src/service/sending/sender.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 61cb9f6d..54302fd5 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -324,8 +324,10 @@ fn select_edus_presence( } } - let presence_content = Edu::Presence(PresenceContent::new(presence_updates)); - events.push(serde_json::to_vec(&presence_content).expect("PresenceEvent can be serialized")); + if !presence_updates.is_empty() { + let presence_content = Edu::Presence(PresenceContent::new(presence_updates)); + events.push(serde_json::to_vec(&presence_content).expect("PresenceEvent can be serialized")); + } Ok(true) } From 04e3de08eba355557bfa7ace886e7f4231fb7a26 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 4 Jul 2024 23:53:30 +0000 Subject: [PATCH 017/158] add log to error functors for Result::map_or_else Signed-off-by: Jason Volk --- src/core/error.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/core/error.rs b/src/core/error.rs index f977803d..ea7acda5 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -19,7 +19,8 @@ use ruma::{ OwnedServerName, }; use thiserror::Error; -use tracing::error; + +use crate::{debug_error, error}; #[derive(Error)] pub enum Error { @@ -126,6 +127,18 @@ impl fmt::Debug for Error { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{self}") } } +#[inline] +pub fn log(e: Error) { + error!("{e}"); + drop(e); +} + +#[inline] +pub fn debug_log(e: Error) { + debug_error!("{e}"); + drop(e); +} + #[derive(Clone)] pub struct RumaResponse(pub T); From 8350aced396e610f70d2cc7a1997e81a8b5407ee Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 5 Jul 2024 01:27:13 +0000 Subject: [PATCH 018/158] add ctrl-\ quit support to admin console Signed-off-by: Jason Volk --- Cargo.lock | 3 +-- Cargo.toml | 3 ++- src/service/admin/console.rs | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f89c9f35..563b87ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3274,8 +3274,7 @@ checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" [[package]] name = "rustyline-async" version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6eb06391513b2184f0a5405c11a4a0a5302e8be442f4c5c35267187c2b37d5" +source = "git+https://github.com/girlbossceo/rustyline-async?rev=de26100b0db03e419a3d8e1dd26895d170d1fe50#de26100b0db03e419a3d8e1dd26895d170d1fe50" dependencies = [ "crossterm", "futures-channel", diff --git a/Cargo.toml b/Cargo.toml index 44f43849..166a359d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -404,7 +404,8 @@ features = [ ] [workspace.dependencies.rustyline-async] -version = "0.4.2" +git = "https://github.com/girlbossceo/rustyline-async" +rev = "de26100b0db03e419a3d8e1dd26895d170d1fe50" [workspace.dependencies.termimad] version = "0.29.4" diff --git a/src/service/admin/console.rs b/src/service/admin/console.rs index c0562c52..3d96e078 100644 --- a/src/service/admin/console.rs +++ b/src/service/admin/console.rs @@ -97,6 +97,7 @@ impl Console { ReadlineEvent::Line(string) => self.clone().handle(string).await, ReadlineEvent::Interrupted => continue, ReadlineEvent::Eof => break, + ReadlineEvent::Quit => services().server.shutdown().unwrap_or_else(error::log), }, Err(error) => match error { ReadlineError::Closed => break, From 38a24e017089b859b68463cec721f12bd2214567 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 5 Jul 2024 01:31:06 +0000 Subject: [PATCH 019/158] remove unused deps in member crates Signed-off-by: Jason Volk --- Cargo.lock | 1 - src/router/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 563b87ae..446a4518 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -711,7 +711,6 @@ dependencies = [ "axum-server", "axum-server-dual-protocol", "bytes", - "clap", "conduit_admin", "conduit_api", "conduit_core", diff --git a/src/router/Cargo.toml b/src/router/Cargo.toml index 2fac3858..ecb09dd7 100644 --- a/src/router/Cargo.toml +++ b/src/router/Cargo.toml @@ -61,7 +61,6 @@ tokio.workspace = true tower.workspace = true tracing.workspace = true bytes.workspace = true -clap.workspace = true http-body-util.workspace = true http.workspace = true hyper.workspace = true From 0e580292a6fe4da501347fa7d51be3809759bbd6 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 5 Jul 2024 01:44:43 +0000 Subject: [PATCH 020/158] encap admin handler init/fini in crate Signed-off-by: Jason Volk --- src/admin/mod.rs | 13 ++++++++++++- src/router/run.rs | 4 ++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/admin/mod.rs b/src/admin/mod.rs index f2034724..7bb22418 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -17,7 +17,6 @@ extern crate conduit_core as conduit; extern crate conduit_service as service; pub(crate) use conduit::{mod_ctor, mod_dtor, Result}; -pub use handler::handle; pub(crate) use service::{services, user_is_local}; pub(crate) use crate::{ @@ -28,6 +27,18 @@ pub(crate) use crate::{ mod_ctor! {} mod_dtor! {} +/// Install the admin command handler +#[allow(clippy::let_underscore_must_use)] +pub async fn init() { + _ = services().admin.handle.lock().await.insert(handler::handle); +} + +/// Uninstall the admin command handler +#[allow(clippy::let_underscore_must_use)] +pub async fn fini() { + _ = services().admin.handle.lock().await.take(); +} + #[cfg(test)] mod test { use clap::Parser; diff --git a/src/router/run.rs b/src/router/run.rs index fb59c797..4f7853d8 100644 --- a/src/router/run.rs +++ b/src/router/run.rs @@ -22,7 +22,7 @@ pub(crate) async fn run(server: Arc) -> Result<(), Error> { let app = layers::build(&server)?; // Install the admin room callback here for now - _ = services().admin.handle.lock().await.insert(admin::handle); + admin::init().await; // Setup shutdown/signal handling let handle = ServerHandle::new(); @@ -39,7 +39,7 @@ pub(crate) async fn run(server: Arc) -> Result<(), Error> { _ = sigs.await; // Remove the admin room callback - _ = services().admin.handle.lock().await.take(); + admin::fini().await; debug_info!("Finished"); res From b31e81a469567a9064d9a96cfa8cf4507e2efb42 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 5 Jul 2024 07:01:11 +0000 Subject: [PATCH 021/158] add common_prefix string util Signed-off-by: Jason Volk --- src/core/utils/string.rs | 19 +++++++++++++++++++ src/core/utils/tests.rs | 27 +++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/src/core/utils/string.rs b/src/core/utils/string.rs index 1a5b6b72..1f2a6572 100644 --- a/src/core/utils/string.rs +++ b/src/core/utils/string.rs @@ -2,6 +2,25 @@ use crate::Result; pub const EMPTY: &str = ""; +/// Find the common prefix from a collection of strings and return a slice +/// ``` +/// use conduit_core::utils::string::common_prefix; +/// let input = ["conduwuit", "conduit", "construct"]; +/// common_prefix(&input) == "con"; +/// ``` +#[must_use] +pub fn common_prefix<'a>(choice: &'a [&str]) -> &'a str { + choice.first().map_or(EMPTY, move |best| { + choice.iter().skip(1).fold(*best, |best, choice| { + &best[0..choice + .chars() + .zip(best.chars()) + .take_while(|&(a, b)| a == b) + .count()] + }) + }) +} + #[inline] #[must_use] pub fn split_once_infallible<'a>(input: &'a str, delim: &'_ str) -> (&'a str, &'a str) { diff --git a/src/core/utils/tests.rs b/src/core/utils/tests.rs index b226bd41..f5cd0a07 100644 --- a/src/core/utils/tests.rs +++ b/src/core/utils/tests.rs @@ -35,3 +35,30 @@ fn increment_wrap() { let res = u64::from_be_bytes(bytes); assert_eq!(res, 0); } + +#[test] +fn common_prefix() { + use utils::string; + + let input = ["conduwuit", "conduit", "construct"]; + let output = string::common_prefix(&input); + assert_eq!(output, "con"); +} + +#[test] +fn common_prefix_empty() { + use utils::string; + + let input = ["abcdefg", "hijklmn", "opqrstu"]; + let output = string::common_prefix(&input); + assert_eq!(output, ""); +} + +#[test] +fn common_prefix_none() { + use utils::string; + + let input = []; + let output = string::common_prefix(&input); + assert_eq!(output, ""); +} From 5254eb4f721079a9f6666b274b109bad618a6203 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 5 Jul 2024 07:52:05 +0000 Subject: [PATCH 022/158] add basic tab completion to console Signed-off-by: Jason Volk --- src/admin/handler.rs | 150 +++++++++++++++++++++++------------ src/admin/mod.rs | 12 +++ src/service/admin/console.rs | 8 ++ src/service/admin/mod.rs | 16 +++- 4 files changed, 134 insertions(+), 52 deletions(-) diff --git a/src/admin/handler.rs b/src/admin/handler.rs index 0bd8af9f..1466c040 100644 --- a/src/admin/handler.rs +++ b/src/admin/handler.rs @@ -1,6 +1,6 @@ use std::time::Instant; -use clap::Parser; +use clap::{CommandFactory, Parser}; use conduit::trace; use ruma::events::{ relation::InReplyTo, @@ -9,7 +9,7 @@ use ruma::events::{ extern crate conduit_service as service; -use conduit::Result; +use conduit::{utils::string::common_prefix, Result}; pub(crate) use service::admin::{Command, Service}; use service::admin::{CommandOutput, CommandResult, HandlerResult}; @@ -62,7 +62,10 @@ pub(crate) enum AdminCommand { } #[must_use] -pub fn handle(command: Command) -> HandlerResult { Box::pin(handle_command(command)) } +pub(crate) fn handle(command: Command) -> HandlerResult { Box::pin(handle_command(command)) } + +#[must_use] +pub(crate) fn complete(line: &str) -> String { complete_admin_command(AdminCommand::command(), line) } #[tracing::instrument(skip_all, name = "admin")] async fn handle_command(command: Command) -> CommandResult { @@ -105,54 +108,6 @@ async fn process_admin_message(msg: String) -> CommandOutput { } } -// Parse chat messages from the admin room into an AdminCommand object -fn parse_admin_command(command_line: &str) -> Result { - let mut argv = command_line.split_whitespace().collect::>(); - - // Remove any escapes that came with a server-side escape command - if !argv.is_empty() && argv[0].ends_with("admin") { - argv[0] = argv[0].trim_start_matches('\\'); - } - - // First indice has to be "admin" but for console convenience we add it here - let server_user = services().globals.server_user.as_str(); - if !argv.is_empty() && !argv[0].ends_with("admin") && !argv[0].starts_with(server_user) { - argv.insert(0, "admin"); - } - - // Replace `help command` with `command --help` - // Clap has a help subcommand, but it omits the long help description. - if argv.len() > 1 && argv[1] == "help" { - argv.remove(1); - argv.push("--help"); - } - - // Backwards compatibility with `register_appservice`-style commands - let command_with_dashes_argv1; - if argv.len() > 1 && argv[1].contains('_') { - command_with_dashes_argv1 = argv[1].replace('_', "-"); - argv[1] = &command_with_dashes_argv1; - } - - // Backwards compatibility with `register_appservice`-style commands - let command_with_dashes_argv2; - if argv.len() > 2 && argv[2].contains('_') { - command_with_dashes_argv2 = argv[2].replace('_', "-"); - argv[2] = &command_with_dashes_argv2; - } - - // if the user is using the `query` command (argv[1]), replace the database - // function/table calls with underscores to match the codebase - let command_with_dashes_argv3; - if argv.len() > 3 && argv[1].eq("query") { - command_with_dashes_argv3 = argv[3].replace('_', "-"); - argv[3] = &command_with_dashes_argv3; - } - - trace!(?command_line, ?argv, "parse"); - AdminCommand::try_parse_from(argv).map_err(|error| error.to_string()) -} - #[tracing::instrument(skip_all, name = "command")] async fn process_admin_command(command: AdminCommand, body: Vec<&str>) -> Result { let reply_message_content = match command { @@ -169,3 +124,96 @@ async fn process_admin_command(command: AdminCommand, body: Vec<&str>) -> Result Ok(reply_message_content) } + +// Parse chat messages from the admin room into an AdminCommand object +fn parse_admin_command(command_line: &str) -> Result { + let argv = parse_command_line(command_line); + AdminCommand::try_parse_from(argv).map_err(|error| error.to_string()) +} + +fn complete_admin_command(mut cmd: clap::Command, line: &str) -> String { + let mut ret = Vec::::new(); + let argv = parse_command_line(line); + 'token: for token in argv.into_iter().skip(1) { + let mut choice = Vec::new(); + let cmd_ = cmd.clone(); + for sub in cmd_.get_subcommands() { + let name = sub.get_name(); + if *name == token { + // token already complete; recurse to subcommand + ret.push(token); + cmd.clone_from(sub); + continue 'token; + } + if name.starts_with(&token) { + // partial match; add to choices + choice.push(name); + } + } + + if choice.is_empty() { + // Nothing found, return original string + ret.push(token); + } else if choice.len() == 1 { + // One choice. Add extra space because it's complete + ret.push((*choice.first().expect("only choice")).to_owned()); + ret.push(String::new()); + } else { + // Find the common prefix + ret.push(common_prefix(&choice).into()); + } + + // Return from completion + return ret.join(" "); + } + + // Return from no completion. Needs a space though. + let mut ret = ret.join(" "); + ret.push(' '); + ret +} + +// Parse chat messages from the admin room into an AdminCommand object +fn parse_command_line(command_line: &str) -> Vec { + let mut argv = command_line + .split_whitespace() + .map(str::to_owned) + .collect::>(); + + // Remove any escapes that came with a server-side escape command + if !argv.is_empty() && argv[0].ends_with("admin") { + argv[0] = argv[0].trim_start_matches('\\').into(); + } + + // First indice has to be "admin" but for console convenience we add it here + let server_user = services().globals.server_user.as_str(); + if !argv.is_empty() && !argv[0].ends_with("admin") && !argv[0].starts_with(server_user) { + argv.insert(0, "admin".to_owned()); + } + + // Replace `help command` with `command --help` + // Clap has a help subcommand, but it omits the long help description. + if argv.len() > 1 && argv[1] == "help" { + argv.remove(1); + argv.push("--help".to_owned()); + } + + // Backwards compatibility with `register_appservice`-style commands + if argv.len() > 1 && argv[1].contains('_') { + argv[1] = argv[1].replace('_', "-"); + } + + // Backwards compatibility with `register_appservice`-style commands + if argv.len() > 2 && argv[2].contains('_') { + argv[2] = argv[2].replace('_', "-"); + } + + // if the user is using the `query` command (argv[1]), replace the database + // function/table calls with underscores to match the codebase + if argv.len() > 3 && argv[1].eq("query") { + argv[3] = argv[3].replace('_', "-"); + } + + trace!(?command_line, ?argv, "parse"); + argv +} diff --git a/src/admin/mod.rs b/src/admin/mod.rs index 7bb22418..2de42420 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -30,6 +30,12 @@ mod_dtor! {} /// Install the admin command handler #[allow(clippy::let_underscore_must_use)] pub async fn init() { + _ = services() + .admin + .complete + .write() + .expect("locked for writing") + .insert(handler::complete); _ = services().admin.handle.lock().await.insert(handler::handle); } @@ -37,6 +43,12 @@ pub async fn init() { #[allow(clippy::let_underscore_must_use)] pub async fn fini() { _ = services().admin.handle.lock().await.take(); + _ = services() + .admin + .complete + .write() + .expect("locked for writing") + .take(); } #[cfg(test)] diff --git a/src/service/admin/console.rs b/src/service/admin/console.rs index 3d96e078..0a200cae 100644 --- a/src/service/admin/console.rs +++ b/src/service/admin/console.rs @@ -118,6 +118,7 @@ impl Console { let _suppression = log::Suppress::new(&services().server); let (mut readline, _writer) = Readline::new(PROMPT.to_owned())?; + readline.set_tab_completer(Self::tab_complete); self.set_history(&mut readline); let future = readline.readline(); @@ -185,6 +186,13 @@ impl Console { history.push_front(line); history.truncate(HISTORY_LIMIT); } + + fn tab_complete(line: &str) -> String { + services() + .admin + .complete_command(line) + .unwrap_or_else(|| line.to_owned()) + } } fn configure_output(mut output: MadSkin) -> MadSkin { diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index ed2ac725..d01a6f55 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -2,7 +2,11 @@ pub mod console; mod create; mod grant; -use std::{future::Future, pin::Pin, sync::Arc}; +use std::{ + future::Future, + pin::Pin, + sync::{Arc, RwLock as StdRwLock}, +}; use async_trait::async_trait; use conduit::{error, utils::mutex_map, Error, Result}; @@ -27,12 +31,14 @@ pub type CommandOutput = Option; pub type CommandResult = Result; pub type HandlerResult = Pin + Send>>; pub type Handler = fn(Command) -> HandlerResult; +pub type Completer = fn(&str) -> String; pub struct Service { sender: Sender, receiver: Mutex>, handler_join: Mutex>>, pub handle: Mutex>, + pub complete: StdRwLock>, #[cfg(feature = "console")] pub console: Arc, } @@ -52,6 +58,7 @@ impl crate::Service for Service { receiver: Mutex::new(receiver), handler_join: Mutex::new(None), handle: Mutex::new(None), + complete: StdRwLock::new(None), #[cfg(feature = "console")] console: console::Console::new(), })) @@ -127,6 +134,13 @@ impl Service { .await } + pub fn complete_command(&self, command: &str) -> Option { + self.complete + .read() + .expect("locked for reading") + .map(|complete| complete(command)) + } + async fn send(&self, message: Command) { debug_assert!(!self.sender.is_closed(), "channel closed"); self.sender.send_async(message).await.expect("message sent"); From a3638dbb15fc9e320f24f6e2d4e3fab2aac08856 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 5 Jul 2024 08:39:37 +0000 Subject: [PATCH 023/158] use rwlock for command handler. Signed-off-by: Jason Volk --- src/admin/mod.rs | 9 +++++++-- src/service/admin/mod.rs | 11 +++++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/admin/mod.rs b/src/admin/mod.rs index 2de42420..f2e35d80 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -36,13 +36,18 @@ pub async fn init() { .write() .expect("locked for writing") .insert(handler::complete); - _ = services().admin.handle.lock().await.insert(handler::handle); + _ = services() + .admin + .handle + .write() + .await + .insert(handler::handle); } /// Uninstall the admin command handler #[allow(clippy::let_underscore_must_use)] pub async fn fini() { - _ = services().admin.handle.lock().await.take(); + _ = services().admin.handle.write().await.take(); _ = services() .admin .complete diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index d01a6f55..ca0e551b 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -21,7 +21,10 @@ use ruma::{ OwnedEventId, OwnedRoomId, RoomId, UserId, }; use serde_json::value::to_raw_value; -use tokio::{sync::Mutex, task::JoinHandle}; +use tokio::{ + sync::{Mutex, RwLock}, + task::JoinHandle, +}; use crate::{pdu::PduBuilder, services, user_is_local, PduEvent}; @@ -37,7 +40,7 @@ pub struct Service { sender: Sender, receiver: Mutex>, handler_join: Mutex>>, - pub handle: Mutex>, + pub handle: RwLock>, pub complete: StdRwLock>, #[cfg(feature = "console")] pub console: Arc, @@ -57,7 +60,7 @@ impl crate::Service for Service { sender, receiver: Mutex::new(receiver), handler_join: Mutex::new(None), - handle: Mutex::new(None), + handle: RwLock::new(None), complete: StdRwLock::new(None), #[cfg(feature = "console")] console: console::Console::new(), @@ -175,7 +178,7 @@ impl Service { } async fn process_command(&self, command: Command) -> CommandResult { - if let Some(handle) = self.handle.lock().await.as_ref() { + if let Some(handle) = self.handle.read().await.as_ref() { handle(command).await } else { Err(Error::Err("Admin module is not loaded.".into())) From 72d9e8ed2bd2f26ce27aef421d2e6a4be36500da Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 7 Jul 2024 14:32:41 -0400 Subject: [PATCH 024/158] bump conduwuit version to 0.4.5 Signed-off-by: strawberry --- Cargo.lock | 14 +++++++------- Cargo.toml | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 446a4518..3415cac0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -570,7 +570,7 @@ checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" [[package]] name = "conduit" -version = "0.4.4" +version = "0.4.5" dependencies = [ "clap", "conduit_admin", @@ -598,7 +598,7 @@ dependencies = [ [[package]] name = "conduit_admin" -version = "0.4.4" +version = "0.4.5" dependencies = [ "clap", "conduit_api", @@ -620,7 +620,7 @@ dependencies = [ [[package]] name = "conduit_api" -version = "0.4.4" +version = "0.4.5" dependencies = [ "axum 0.7.5", "axum-client-ip", @@ -653,7 +653,7 @@ dependencies = [ [[package]] name = "conduit_core" -version = "0.4.4" +version = "0.4.5" dependencies = [ "argon2", "axum 0.7.5", @@ -692,7 +692,7 @@ dependencies = [ [[package]] name = "conduit_database" -version = "0.4.4" +version = "0.4.5" dependencies = [ "conduit_core", "log", @@ -704,7 +704,7 @@ dependencies = [ [[package]] name = "conduit_router" -version = "0.4.4" +version = "0.4.5" dependencies = [ "axum 0.7.5", "axum-client-ip", @@ -736,7 +736,7 @@ dependencies = [ [[package]] name = "conduit_service" -version = "0.4.4" +version = "0.4.5" dependencies = [ "async-trait", "base64 0.22.1", diff --git a/Cargo.toml b/Cargo.toml index 166a359d..29454e60 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ license = "Apache-2.0" readme = "README.md" repository = "https://github.com/girlbossceo/conduwuit" rust-version = "1.77.0" -version = "0.4.4" +version = "0.4.5" [workspace.metadata.crane] name = "conduit" From c1c084dda1c6b3801e81b53ceaa8bfa26a9e9245 Mon Sep 17 00:00:00 2001 From: AlexPewMaster <68469103+AlexPewMaster@users.noreply.github.com> Date: Mon, 8 Jul 2024 18:31:08 +0200 Subject: [PATCH 025/158] Introduce a new way of deploying conduwuit using caddy in Docker --- docs/deploying/docker-compose.with-caddy.yml | 54 ++++++++++++++++++++ docs/deploying/docker.md | 11 +++- 2 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 docs/deploying/docker-compose.with-caddy.yml diff --git a/docs/deploying/docker-compose.with-caddy.yml b/docs/deploying/docker-compose.with-caddy.yml new file mode 100644 index 00000000..41d8856d --- /dev/null +++ b/docs/deploying/docker-compose.with-caddy.yml @@ -0,0 +1,54 @@ +services: + caddy: + # This compose file uses caddy-docker-proxy as the reverse proxy for conduwuit! + # For more info, visit https://github.com/lucaslorentz/caddy-docker-proxy + image: lucaslorentz/caddy-docker-proxy:ci-alpine + ports: + - 80:80 + - 443:443 + environment: + - CADDY_INGRESS_NETWORKS=caddy + networks: + - caddy + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./data:/data + restart: unless-stopped + labels: + caddy: example.com + caddy.0_respond: /.well-known/matrix/server {"m.server":"matrix.example.com:443"} + caddy.1_respond: /.well-known/matrix/client {"m.server":{"base_url":"https://matrix.example.com"},"m.homeserver":{"base_url":"https://matrix.example.com"},"org.matrix.msc3575.proxy":{"url":"https://matrix.example.com"}} + + homeserver: + ### If you already built the conduwuit image with 'docker build' or want to use a registry image, + ### then you are ready to go. + image: girlbossceo/conduwuit:latest + restart: unless-stopped + volumes: + - db:/var/lib/conduwuit + #- ./conduwuit.toml:/etc/conduwuit.toml + environment: + CONDUWUIT_SERVER_NAME: example.com # EDIT THIS + CONDUWUIT_DATABASE_PATH: /var/lib/conduwuit + CONDUWUIT_DATABASE_BACKEND: rocksdb + CONDUWUIT_PORT: 6167 + CONDUWUIT_MAX_REQUEST_SIZE: 20_000_000 # in bytes, ~20 MB + CONDUWUIT_ALLOW_REGISTRATION: 'true' + CONDUWUIT_ALLOW_FEDERATION: 'true' + CONDUWUIT_ALLOW_CHECK_FOR_UPDATES: 'true' + CONDUWUIT_TRUSTED_SERVERS: '["matrix.org"]' + #CONDUWUIT_LOG: warn,state_res=warn + CONDUWUIT_ADDRESS: 0.0.0.0 + #CONDUWUIT_CONFIG: './conduwuit.toml' # Uncomment if you mapped config toml above + networks: + - caddy + labels: + caddy: matrix.example.com + caddy.reverse_proxy: "{{upstreams 6167}}" + +volumes: + db: + +networks: + caddy: + external: true diff --git a/docs/deploying/docker.md b/docs/deploying/docker.md index 5cb89308..ffbad5b4 100644 --- a/docs/deploying/docker.md +++ b/docs/deploying/docker.md @@ -59,13 +59,22 @@ If the `docker run` command is not for you or your setup, you can also use one o Depending on your proxy setup, you can use one of the following files; - If you already have a `traefik` instance set up, use [`docker-compose.for-traefik.yml`](docker-compose.for-traefik.yml) -- If you don't have a `traefik` instance set up (or any other reverse proxy), use [`docker-compose.with-traefik.yml`](docker-compose.with-traefik.yml) +- If you don't have a `traefik` instance set up and would like to use it, use [`docker-compose.with-traefik.yml`](docker-compose.with-traefik.yml) +- If you want a setup that works out of the box with `caddy-docker-proxy`, use [`docker-compose.with-caddy.yml`](docker-compose.with-caddy.yml) and replace all `example.com` placeholders with your own domain - For any other reverse proxy, use [`docker-compose.yml`](docker-compose.yml) When picking the traefik-related compose file, rename it so it matches `docker-compose.yml`, and rename the override file to `docker-compose.override.yml`. Edit the latter with the values you want for your server. +When picking the `caddy-docker-proxy` compose file, it's important to first create the `caddy` network before spinning up the containers: + +```bash +docker network create caddy +``` + +After that, you can rename it so it matches `docker-compose.yml` and spin up the containers! + Additional info about deploying conduwuit can be found [here](generic.md). ### Build From a309ef55c910351368687ce02080bddf35d4f0bb Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 6 Jul 2024 13:53:21 +0000 Subject: [PATCH 026/158] restore signal state after channel failures Signed-off-by: Jason Volk --- src/core/server.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/core/server.rs b/src/core/server.rs index e76e4d57..575924d3 100644 --- a/src/core/server.rs +++ b/src/core/server.rs @@ -76,7 +76,10 @@ impl Server { return Err(Error::Err("Shutdown already in progress".into())); } - self.signal("SIGINT") + self.signal("SIGINT").inspect_err(|_| { + self.stopping.store(false, Ordering::Release); + self.reloading.store(false, Ordering::Release); + }) } pub fn restart(&self) -> Result<()> { @@ -85,6 +88,7 @@ impl Server { } self.shutdown() + .inspect_err(|_| self.restarting.store(false, Ordering::Release)) } pub fn shutdown(&self) -> Result<()> { @@ -93,6 +97,7 @@ impl Server { } self.signal("SIGTERM") + .inspect_err(|_| self.stopping.store(false, Ordering::Release)) } pub fn signal(&self, sig: &'static str) -> Result<()> { From 24b37e03a01d0df5d53775f2519523fa7b23333b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 6 Jul 2024 06:47:42 +0000 Subject: [PATCH 027/158] add pretty time util Signed-off-by: Jason Volk --- src/admin/server/commands.rs | 17 +++----- src/core/utils/time.rs | 79 +++++++++++++++++++++++++++++++++++- 2 files changed, 83 insertions(+), 13 deletions(-) diff --git a/src/admin/server/commands.rs b/src/admin/server/commands.rs index 2ab71033..06007fb8 100644 --- a/src/admin/server/commands.rs +++ b/src/admin/server/commands.rs @@ -1,24 +1,17 @@ -use conduit::{warn, Error, Result}; +use conduit::{utils::time, warn, Error, Result}; use ruma::events::room::message::RoomMessageEventContent; use crate::services; pub(super) async fn uptime(_body: Vec<&str>) -> Result { - let seconds = services() + let elapsed = services() .server .started .elapsed() - .expect("standard duration") - .as_secs(); - let result = format!( - "up {} days, {} hours, {} minutes, {} seconds.", - seconds / 86400, - (seconds % 86400) / 60 / 60, - (seconds % 3600) / 60, - seconds % 60, - ); + .expect("standard duration"); - Ok(RoomMessageEventContent::notice_plain(result)) + let result = time::pretty(elapsed); + Ok(RoomMessageEventContent::notice_plain(format!("{result}."))) } pub(super) async fn show_config(_body: Vec<&str>) -> Result { diff --git a/src/core/utils/time.rs b/src/core/utils/time.rs index 7de00e9e..39ceb3c3 100644 --- a/src/core/utils/time.rs +++ b/src/core/utils/time.rs @@ -1,4 +1,6 @@ -use std::time::{SystemTime, UNIX_EPOCH}; +#![allow(clippy::enum_glob_use)] + +use std::time::{Duration, SystemTime, UNIX_EPOCH}; #[inline] #[must_use] @@ -26,3 +28,78 @@ pub fn format(ts: SystemTime, str: &str) -> String { let dt: DateTime = ts.into(); dt.format(str).to_string() } + +#[must_use] +pub fn pretty(d: Duration) -> String { + use Unit::*; + + let fmt = |w, f, u| format!("{w}.{f} {u}"); + let gen64 = |w, f, u| fmt(w, (f * 100.0) as u64, u); + let gen128 = |w, f, u| gen64(u64::try_from(w).expect("u128 to u64"), f, u); + match whole_and_frac(d) { + (Days(whole), frac) => gen64(whole, frac, "days"), + (Hours(whole), frac) => gen64(whole, frac, "hours"), + (Mins(whole), frac) => gen64(whole, frac, "minutes"), + (Secs(whole), frac) => gen64(whole, frac, "seconds"), + (Millis(whole), frac) => gen128(whole, frac, "milliseconds"), + (Micros(whole), frac) => gen128(whole, frac, "microseconds"), + (Nanos(whole), frac) => gen128(whole, frac, "nanoseconds"), + } +} + +/// Return a pair of (whole part, frac part) from a duration where. The whole +/// part is the largest Unit containing a non-zero value, the frac part is a +/// rational remainder left over. +#[must_use] +pub fn whole_and_frac(d: Duration) -> (Unit, f64) { + use Unit::*; + + let whole = whole_unit(d); + ( + whole, + match whole { + Days(_) => (d.as_secs() % 86_400) as f64 / 86_400.0, + Hours(_) => (d.as_secs() % 3_600) as f64 / 3_600.0, + Mins(_) => (d.as_secs() % 60) as f64 / 60.0, + Secs(_) => f64::from(d.subsec_millis()) / 1000.0, + Millis(_) => f64::from(d.subsec_micros()) / 1000.0, + Micros(_) => f64::from(d.subsec_nanos()) / 1000.0, + Nanos(_) => 0.0, + }, + ) +} + +/// Return the largest Unit which represents the duration. The value is +/// rounded-down, but never zero. +#[must_use] +pub fn whole_unit(d: Duration) -> Unit { + use Unit::*; + + match d.as_secs() { + 86_400.. => Days(d.as_secs() / 86_400), + 3_600..=86_399 => Hours(d.as_secs() / 3_600), + 60..=3_599 => Mins(d.as_secs() / 60), + + _ => match d.as_micros() { + 1_000_000.. => Secs(d.as_secs()), + 1_000..=999_999 => Millis(d.subsec_millis().into()), + + _ => match d.as_nanos() { + 1_000.. => Micros(d.subsec_micros().into()), + + _ => Nanos(d.subsec_nanos().into()), + }, + }, + } +} + +#[derive(Eq, PartialEq, Clone, Copy, Debug)] +pub enum Unit { + Days(u64), + Hours(u64), + Mins(u64), + Secs(u64), + Millis(u128), + Micros(u128), + Nanos(u128), +} From a388c2e06edf48671d244e73777fb5f3be8c65bc Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 6 Jul 2024 06:50:29 +0000 Subject: [PATCH 028/158] allow clippy::enum_glob_use Signed-off-by: Jason Volk --- Cargo.toml | 1 + src/core/utils/time.rs | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 29454e60..f6b23102 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -741,6 +741,7 @@ cast_possible_truncation = { level = "allow", priority = 1 } cast_precision_loss = { level = "allow", priority = 1 } cast_sign_loss = { level = "allow", priority = 1 } doc_markdown = { level = "allow", priority = 1 } +enum_glob_use = { level = "allow", priority = 1 } error_impl_error = { level = "allow", priority = 1 } expect_used = { level = "allow", priority = 1 } if_not_else = { level = "allow", priority = 1 } diff --git a/src/core/utils/time.rs b/src/core/utils/time.rs index 39ceb3c3..2fe90b31 100644 --- a/src/core/utils/time.rs +++ b/src/core/utils/time.rs @@ -1,5 +1,3 @@ -#![allow(clippy::enum_glob_use)] - use std::time::{Duration, SystemTime, UNIX_EPOCH}; #[inline] From 611f09829efd839ea290e8806ac43dfcd19aa0f6 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 5 Jul 2024 08:48:24 +0000 Subject: [PATCH 029/158] use shorthand constraint syntax and formatting Signed-off-by: Jason Volk --- src/core/utils/defer.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/core/utils/defer.rs b/src/core/utils/defer.rs index 2762d4fa..9d42e679 100644 --- a/src/core/utils/defer.rs +++ b/src/core/utils/defer.rs @@ -1,17 +1,11 @@ #[macro_export] macro_rules! defer { ($body:block) => { - struct _Defer_ - where - F: FnMut(), - { + struct _Defer_ { closure: F, } - impl Drop for _Defer_ - where - F: FnMut(), - { + impl Drop for _Defer_ { fn drop(&mut self) { (self.closure)(); } } From 5e72d368006247e664f5c9c2fe52d21aa91c2f88 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 7 Jul 2024 03:36:50 +0000 Subject: [PATCH 030/158] add math utils; integrate checked expression macros Signed-off-by: Jason Volk --- Cargo.lock | 10 ++++++++++ Cargo.toml | 3 +++ src/core/Cargo.toml | 1 + src/core/error.rs | 8 ++++++++ src/core/utils/math.rs | 26 ++++++++++++++++++++++++++ src/core/utils/mod.rs | 1 + src/core/utils/tests.rs | 19 +++++++++++++++++++ 7 files changed, 68 insertions(+) create mode 100644 src/core/utils/math.rs diff --git a/Cargo.lock b/Cargo.lock index 3415cac0..7152c66d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -504,6 +504,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "checked_ops" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b491d76efc1d99d74de3c8529bee64c62312c275c7eb124f9185291de45801d5" +dependencies = [ + "num-traits", +] + [[package]] name = "chrono" version = "0.4.38" @@ -658,6 +667,7 @@ dependencies = [ "argon2", "axum 0.7.5", "bytes", + "checked_ops", "chrono", "either", "figment", diff --git a/Cargo.toml b/Cargo.toml index f6b23102..c93c6736 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -411,6 +411,9 @@ rev = "de26100b0db03e419a3d8e1dd26895d170d1fe50" version = "0.29.4" default-features = false +[workspace.dependencies.checked_ops] +version = "0.1" + # # Patches diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index 2c0fc47d..e47f673e 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -53,6 +53,7 @@ sha256_media = [] argon2.workspace = true axum.workspace = true bytes.workspace = true +checked_ops.workspace = true chrono.workspace = true either.workspace = true figment.workspace = true diff --git a/src/core/error.rs b/src/core/error.rs index ea7acda5..63729f31 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -35,6 +35,12 @@ pub enum Error { FromUtf8Error(#[from] std::string::FromUtf8Error), #[error("{0}")] TryFromSliceError(#[from] std::array::TryFromSliceError), + #[error("{0}")] + TryFromIntError(#[from] std::num::TryFromIntError), + #[error("{0}")] + ParseIntError(#[from] std::num::ParseIntError), + #[error("{0}")] + ParseFloatError(#[from] std::num::ParseFloatError), // third-party #[error("Regex error: {0}")] @@ -63,6 +69,8 @@ pub enum Error { InconsistentRoomState(&'static str, ruma::OwnedRoomId), // conduwuit + #[error("Arithmetic operation failed: {0}")] + Arithmetic(&'static str), #[error("There was a problem with your configuration: {0}")] BadConfig(String), #[error("{0}")] diff --git a/src/core/utils/math.rs b/src/core/utils/math.rs new file mode 100644 index 00000000..d5dbf3a6 --- /dev/null +++ b/src/core/utils/math.rs @@ -0,0 +1,26 @@ +use std::{cmp, time::Duration}; + +pub use checked_ops::checked_ops; + +/// Checked arithmetic expression. Returns a Result +#[macro_export] +macro_rules! checked { + ($($input:tt)*) => { + $crate::utils::math::checked_ops!($($input)*) + .ok_or_else(|| $crate::Error::Arithmetic("operation overflowed or result invalid")) + } +} + +/// in release-mode. Use for performance when the expression is obviously safe. +/// The check remains in debug-mode for regression analysis. +#[cfg(not(debug_assertions))] +#[macro_export] +macro_rules! validated { + ($($input:tt)*) => { Ok($($input)*) } +} + +#[cfg(debug_assertions)] +#[macro_export] +macro_rules! validated { + ($($input:tt)*) => { $crate::checked!($($input)*) } +} diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index f9f1b87e..2b79c3c4 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -5,6 +5,7 @@ pub mod defer; pub mod hash; pub mod html; pub mod json; +pub mod math; pub mod mutex_map; pub mod rand; pub mod string; diff --git a/src/core/utils/tests.rs b/src/core/utils/tests.rs index f5cd0a07..239e27e9 100644 --- a/src/core/utils/tests.rs +++ b/src/core/utils/tests.rs @@ -62,3 +62,22 @@ fn common_prefix_none() { let output = string::common_prefix(&input); assert_eq!(output, ""); } + +#[test] +fn checked_add() { + use utils::math::checked; + + let a = 1234; + let res = checked!(a + 1).unwrap(); + assert_eq!(res, 1235); +} + +#[test] +#[should_panic(expected = "overflow")] +fn checked_add_overflow() { + use utils::math::checked; + + let a: u64 = u64::MAX; + let res = checked!(a + 1).expect("overflow"); + assert_eq!(res, 0); +} From 52a561ff9e8196794995ccc2556c9976a636fb8b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 7 Jul 2024 03:39:35 +0000 Subject: [PATCH 031/158] abstract expoential backoff to math utils. Signed-off-by: Jason Volk --- src/api/client/keys.rs | 21 +++++-------- src/api/client/membership.rs | 21 ++++++------- src/core/utils/math.rs | 18 +++++++++++ src/service/rooms/event_handler/mod.rs | 41 +++++++++++--------------- src/service/sending/sender.rs | 12 ++++---- 5 files changed, 60 insertions(+), 53 deletions(-) diff --git a/src/api/client/keys.rs b/src/api/client/keys.rs index 6f089875..7bb02a60 100644 --- a/src/api/client/keys.rs +++ b/src/api/client/keys.rs @@ -1,9 +1,9 @@ use std::{ - cmp, collections::{hash_map, BTreeMap, HashMap, HashSet}, - time::{Duration, Instant}, + time::Instant, }; +use conduit::{utils, utils::math::continue_exponential_backoff_secs, Error, Result}; use futures_util::{stream::FuturesUnordered, StreamExt}; use ruma::{ api::{ @@ -18,15 +18,11 @@ use ruma::{ DeviceKeyAlgorithm, OwnedDeviceId, OwnedUserId, UserId, }; use serde_json::json; +use service::user_is_local; use tracing::debug; use super::SESSION_ID_LENGTH; -use crate::{ - service::user_is_local, - services, - utils::{self}, - Error, Result, Ruma, -}; +use crate::{services, Ruma}; /// # `POST /_matrix/client/r0/keys/upload` /// @@ -357,11 +353,10 @@ pub(crate) async fn get_keys_helper bool + Send>( .get(server) { // Exponential backoff - const MAX_DURATION: Duration = Duration::from_secs(60 * 60 * 24); - let min_elapsed_duration = cmp::min(MAX_DURATION, Duration::from_secs(5 * 60) * (*tries) * (*tries)); - - if time.elapsed() < min_elapsed_duration { - debug!("Backing off query from {:?}", server); + const MIN: u64 = 5 * 60; + const MAX: u64 = 60 * 60 * 24; + if continue_exponential_backoff_secs(MIN, MAX, time.elapsed(), *tries) { + debug!("Backing off query from {server:?}"); return (server, Err(Error::BadServerResponse("bad query, still backing off"))); } } diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 07a585fd..e2ba4c9a 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -1,13 +1,16 @@ use std::{ - cmp, collections::{hash_map::Entry, BTreeMap, HashMap, HashSet}, net::IpAddr, sync::Arc, - time::{Duration, Instant}, + time::Instant, }; use axum_client_ip::InsecureClientIp; -use conduit::utils::mutex_map; +use conduit::{ + debug, error, info, trace, utils, + utils::{math::continue_exponential_backoff_secs, mutex_map}, + warn, Error, PduEvent, Result, +}; use ruma::{ api::{ client::{ @@ -35,7 +38,6 @@ use ruma::{ use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; use service::sending::convert_to_outgoing_federation_event; use tokio::sync::RwLock; -use tracing::{debug, error, info, trace, warn}; use crate::{ client::{update_avatar_url, update_displayname}, @@ -43,7 +45,7 @@ use crate::{ pdu::{gen_event_id_canonical_json, PduBuilder}, server_is_ours, user_is_local, }, - services, utils, Error, PduEvent, Result, Ruma, + services, Ruma, }; /// Checks if the room is banned in any way possible and the sender user is not @@ -1363,11 +1365,10 @@ pub async fn validate_and_add_event_id( .get(&event_id) { // Exponential backoff - const MAX_DURATION: Duration = Duration::from_secs(60 * 60 * 24); - let min_elapsed_duration = cmp::min(MAX_DURATION, Duration::from_secs(5 * 60) * (*tries) * (*tries)); - - if time.elapsed() < min_elapsed_duration { - debug!("Backing off from {}", event_id); + const MIN: u64 = 60 * 5; + const MAX: u64 = 60 * 60 * 24; + if continue_exponential_backoff_secs(MIN, MAX, time.elapsed(), *tries) { + debug!("Backing off from {event_id}"); return Err(Error::BadServerResponse("bad event, still backing off")); } } diff --git a/src/core/utils/math.rs b/src/core/utils/math.rs index d5dbf3a6..a77f8e26 100644 --- a/src/core/utils/math.rs +++ b/src/core/utils/math.rs @@ -24,3 +24,21 @@ macro_rules! validated { macro_rules! validated { ($($input:tt)*) => { $crate::checked!($($input)*) } } + +/// Returns false if the exponential backoff has expired based on the inputs +#[inline] +#[must_use] +pub fn continue_exponential_backoff_secs(min: u64, max: u64, elapsed: Duration, tries: u32) -> bool { + let min = Duration::from_secs(min); + let max = Duration::from_secs(max); + continue_exponential_backoff(min, max, elapsed, tries) +} + +/// Returns false if the exponential backoff has expired based on the inputs +#[inline] +#[must_use] +pub fn continue_exponential_backoff(min: Duration, max: Duration, elapsed: Duration, tries: u32) -> bool { + let min = min.saturating_mul(tries).saturating_mul(tries); + let min = cmp::min(min, max); + elapsed < min +} diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 9f50ef58..395b70f2 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -2,14 +2,16 @@ mod parse_incoming_pdu; mod signing_keys; use std::{ - cmp, collections::{hash_map, BTreeMap, HashMap, HashSet}, pin::Pin, sync::Arc, - time::{Duration, Instant}, + time::Instant, }; -use conduit::{debug_error, debug_info, Error, Result}; +use conduit::{ + debug, debug_error, debug_info, error, info, trace, utils::math::continue_exponential_backoff_secs, warn, Error, + Result, +}; use futures_util::Future; pub use parse_incoming_pdu::parse_incoming_pdu; use ruma::{ @@ -29,7 +31,6 @@ use ruma::{ uint, CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, OwnedUserId, RoomId, RoomVersionId, ServerName, }; use tokio::sync::RwLock; -use tracing::{debug, error, info, trace, warn}; use super::state_compressor::CompressedStateEvent; use crate::{pdu, services, PduEvent}; @@ -252,14 +253,12 @@ impl Service { .get(prev_id) { // Exponential backoff - const MAX_DURATION: Duration = Duration::from_secs(60 * 60 * 24); - let min_duration = cmp::min(MAX_DURATION, Duration::from_secs(5 * 60) * (*tries) * (*tries)); - let duration = time.elapsed(); - - if duration < min_duration { + const MIN_DURATION: u64 = 5 * 60; + const MAX_DURATION: u64 = 60 * 60 * 24; + if continue_exponential_backoff_secs(MIN_DURATION, MAX_DURATION, time.elapsed(), *tries) { debug!( - duration = ?duration, - min_duration = ?min_duration, + ?tries, + duration = ?time.elapsed(), "Backing off from prev_event" ); return Ok(()); @@ -1083,12 +1082,10 @@ impl Service { .get(&*next_id) { // Exponential backoff - const MAX_DURATION: Duration = Duration::from_secs(60 * 60 * 24); - let min_elapsed_duration = - cmp::min(MAX_DURATION, Duration::from_secs(5 * 60) * (*tries) * (*tries)); - - if time.elapsed() < min_elapsed_duration { - info!("Backing off from {}", next_id); + const MIN_DURATION: u64 = 5 * 60; + const MAX_DURATION: u64 = 60 * 60 * 24; + if continue_exponential_backoff_secs(MIN_DURATION, MAX_DURATION, time.elapsed(), *tries) { + info!("Backing off from {next_id}"); continue; } } @@ -1191,12 +1188,10 @@ impl Service { .get(&**next_id) { // Exponential backoff - const MAX_DURATION: Duration = Duration::from_secs(60 * 60 * 24); - let min_elapsed_duration = - cmp::min(MAX_DURATION, Duration::from_secs(5 * 60) * (*tries) * (*tries)); - - if time.elapsed() < min_elapsed_duration { - debug!("Backing off from {}", next_id); + const MIN_DURATION: u64 = 5 * 60; + const MAX_DURATION: u64 = 60 * 60 * 24; + if continue_exponential_backoff_secs(MIN_DURATION, MAX_DURATION, time.elapsed(), *tries) { + debug!("Backing off from {next_id}"); continue; } } diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 54302fd5..0f4fa17a 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -3,10 +3,11 @@ use std::{ collections::{BTreeMap, HashMap, HashSet}, fmt::Debug, sync::Arc, - time::{Duration, Instant}, + time::Instant, }; use base64::{engine::general_purpose, Engine as _}; +use conduit::{debug, error, utils::math::continue_exponential_backoff_secs, warn}; use federation::transactions::send_transaction_message; use futures_util::{future::BoxFuture, stream::FuturesUnordered, StreamExt}; use ruma::{ @@ -22,7 +23,6 @@ use ruma::{ ServerName, UInt, }; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; -use tracing::{debug, error, warn}; use super::{appservice, send, Destination, Msg, SendingEvent, Service}; use crate::{presence::Presence, services, user_is_local, utils::calculate_hash, Error, Result}; @@ -216,11 +216,9 @@ impl Service { .and_modify(|e| match e { TransactionStatus::Failed(tries, time) => { // Fail if a request has failed recently (exponential backoff) - let max_duration = Duration::from_secs(services().globals.config.sender_retry_backoff_limit); - let min_duration = Duration::from_secs(services().globals.config.sender_timeout); - let min_elapsed_duration = min_duration * (*tries) * (*tries); - let min_elapsed_duration = cmp::min(min_elapsed_duration, max_duration); - if time.elapsed() < min_elapsed_duration { + let min = services().globals.config.sender_timeout; + let max = services().globals.config.sender_retry_backoff_limit; + if continue_exponential_backoff_secs(min, max, time.elapsed(), *tries) { allow = false; } else { retry = true; From 7397064edd4d1e6d7bc9cb4d2c5b19794e713458 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 7 Jul 2024 04:46:16 +0000 Subject: [PATCH 032/158] fix arithmetic side-effects Signed-off-by: Jason Volk --- Cargo.toml | 2 +- src/admin/debug/commands.rs | 2 +- src/admin/room/room_moderation_commands.rs | 5 +- src/admin/user/commands.rs | 7 ++- src/core/utils/html.rs | 2 +- src/core/utils/math.rs | 10 ++- src/core/utils/rand.rs | 6 +- src/core/utils/tests.rs | 6 +- src/service/globals/data.rs | 5 +- src/service/globals/migrations.rs | 7 ++- src/service/media/mod.rs | 34 +++------- src/service/presence/mod.rs | 73 +++++++++++----------- src/service/rooms/auth_chain/mod.rs | 12 ++-- src/service/rooms/event_handler/mod.rs | 6 +- src/service/rooms/pdu_metadata/mod.rs | 2 +- src/service/rooms/read_receipt/data.rs | 6 +- src/service/rooms/spaces/mod.rs | 5 +- src/service/rooms/state_compressor/data.rs | 11 ++-- src/service/rooms/state_compressor/mod.rs | 16 +++-- src/service/rooms/threads/data.rs | 4 +- src/service/rooms/threads/mod.rs | 2 +- src/service/rooms/timeline/data.rs | 8 ++- src/service/rooms/timeline/mod.rs | 7 ++- src/service/sending/sender.rs | 2 +- src/service/users/data.rs | 13 ++-- 25 files changed, 139 insertions(+), 114 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c93c6736..19ccbb00 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -769,7 +769,7 @@ perf = "warn" ################### #restriction = "warn" -#arithmetic_side_effects = "warn" # TODO +arithmetic_side_effects = "warn" #as_conversions = "warn" # TODO assertions_on_result_states = "warn" dbg_macro = "warn" diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 3efe283b..53009566 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -58,7 +58,7 @@ pub(super) async fn parse_pdu(body: Vec<&str>) -> Result match ruma::signatures::reference_hash(&value, &RoomVersionId::V6) { Ok(hash) => { diff --git a/src/admin/room/room_moderation_commands.rs b/src/admin/room/room_moderation_commands.rs index 39eb7c47..92e3de47 100644 --- a/src/admin/room/room_moderation_commands.rs +++ b/src/admin/room/room_moderation_commands.rs @@ -191,7 +191,10 @@ async fn ban_list_of_rooms(body: Vec<&str>, force: bool, disable_federation: boo )); } - let rooms_s = body.clone().drain(1..body.len() - 1).collect::>(); + let rooms_s = body + .clone() + .drain(1..body.len().saturating_sub(1)) + .collect::>(); let admin_room_alias = &services().globals.admin_alias; diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 6dc60713..884e1d29 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -23,7 +23,7 @@ pub(super) async fn list(_body: Vec<&str>) -> Result { match services().users.list_local_users() { Ok(users) => { let mut plain_msg = format!("Found {} local user account(s):\n```\n", users.len()); - plain_msg += &users.join("\n"); + plain_msg += users.join("\n").as_str(); plain_msg += "\n```"; Ok(RoomMessageEventContent::notice_markdown(plain_msg)) @@ -195,7 +195,10 @@ pub(super) async fn deactivate_all( )); } - let usernames = body.clone().drain(1..body.len() - 1).collect::>(); + let usernames = body + .clone() + .drain(1..body.len().saturating_sub(1)) + .collect::>(); let mut user_ids: Vec = Vec::with_capacity(usernames.len()); let mut admins = Vec::new(); diff --git a/src/core/utils/html.rs b/src/core/utils/html.rs index 3b44a31b..938e50ec 100644 --- a/src/core/utils/html.rs +++ b/src/core/utils/html.rs @@ -26,7 +26,7 @@ impl fmt::Display for Escape<'_> { fmt.write_str(s)?; // NOTE: we only expect single byte characters here - which is fine as long as // we only match single byte characters - last = i + 1; + last = i.saturating_add(1); } if last < s.len() { diff --git a/src/core/utils/math.rs b/src/core/utils/math.rs index a77f8e26..f00b9055 100644 --- a/src/core/utils/math.rs +++ b/src/core/utils/math.rs @@ -16,7 +16,15 @@ macro_rules! checked { #[cfg(not(debug_assertions))] #[macro_export] macro_rules! validated { - ($($input:tt)*) => { Ok($($input)*) } + ($($input:tt)*) => { + //#[allow(clippy::arithmetic_side_effects)] { + //Some($($input)*) + // .ok_or_else(|| $crate::Error::Arithmetic("this error should never been seen")) + //} + + //NOTE: remove me when stmt_expr_attributes is stable + $crate::checked!($($input)*) + } } #[cfg(debug_assertions)] diff --git a/src/core/utils/rand.rs b/src/core/utils/rand.rs index 1ded8a6d..b80671eb 100644 --- a/src/core/utils/rand.rs +++ b/src/core/utils/rand.rs @@ -15,7 +15,11 @@ pub fn string(length: usize) -> String { #[inline] #[must_use] -pub fn timepoint_secs(range: Range) -> SystemTime { SystemTime::now() + secs(range) } +pub fn timepoint_secs(range: Range) -> SystemTime { + SystemTime::now() + .checked_add(secs(range)) + .expect("range does not overflow SystemTime") +} #[must_use] pub fn secs(range: Range) -> Duration { diff --git a/src/core/utils/tests.rs b/src/core/utils/tests.rs index 239e27e9..add15861 100644 --- a/src/core/utils/tests.rs +++ b/src/core/utils/tests.rs @@ -65,7 +65,7 @@ fn common_prefix_none() { #[test] fn checked_add() { - use utils::math::checked; + use crate::checked; let a = 1234; let res = checked!(a + 1).unwrap(); @@ -75,9 +75,9 @@ fn checked_add() { #[test] #[should_panic(expected = "overflow")] fn checked_add_overflow() { - use utils::math::checked; + use crate::checked; - let a: u64 = u64::MAX; + let a = u64::MAX; let res = checked!(a + 1).expect("overflow"); assert_eq!(res, 0); } diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index ecb827fe..083aec3c 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -65,7 +65,10 @@ impl Data { "counter mismatch" ); - *counter = counter.wrapping_add(1); + *counter = counter + .checked_add(1) + .expect("counter must not overflow u64"); + self.global.insert(COUNTER, &counter.to_be_bytes())?; Ok(*counter) diff --git a/src/service/globals/migrations.rs b/src/service/globals/migrations.rs index e171cb9d..3948d1f5 100644 --- a/src/service/globals/migrations.rs +++ b/src/service/globals/migrations.rs @@ -836,11 +836,14 @@ async fn fix_bad_double_separator_in_state_cache(db: &Arc, _config: &C for (mut key, value) in roomuserid_joined.iter() { iter_count = iter_count.saturating_add(1); debug_info!(%iter_count); - let first_sep_index = key.iter().position(|&i| i == 0xFF).unwrap(); + let first_sep_index = key + .iter() + .position(|&i| i == 0xFF) + .expect("found 0xFF delim"); if key .iter() - .get(first_sep_index..=first_sep_index + 1) + .get(first_sep_index..=first_sep_index.saturating_add(1)) .copied() .collect_vec() == vec![0xFF, 0xFF] diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index caa75d4e..3cb8fda8 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -1,10 +1,10 @@ mod data; mod tests; -use std::{collections::HashMap, io::Cursor, path::PathBuf, sync::Arc, time::SystemTime}; +use std::{collections::HashMap, io::Cursor, num::Saturating as Sat, path::PathBuf, sync::Arc, time::SystemTime}; use base64::{engine::general_purpose, Engine as _}; -use conduit::{debug, debug_error, error, utils, Error, Result, Server}; +use conduit::{checked, debug, debug_error, error, utils, Error, Result, Server}; use data::Data; use image::imageops::FilterType; use ruma::{OwnedMxcUri, OwnedUserId}; @@ -305,36 +305,20 @@ impl Service { image.resize_to_fill(width, height, FilterType::CatmullRom) } else { let (exact_width, exact_height) = { - // Copied from image::dynimage::resize_dimensions - // - // https://github.com/image-rs/image/blob/6edf8ae492c4bb1dacb41da88681ea74dab1bab3/src/math/utils.rs#L5-L11 - // Calculates the width and height an image should be - // resized to. This preserves aspect ratio, and based - // on the `fill` parameter will either fill the - // dimensions to fit inside the smaller constraint - // (will overflow the specified bounds on one axis to - // preserve aspect ratio), or will shrink so that both - // dimensions are completely contained within the given - // `width` and `height`, with empty space on one axis. - let ratio = u64::from(original_width) * u64::from(height); - let nratio = u64::from(width) * u64::from(original_height); + let ratio = Sat(original_width) * Sat(height); + let nratio = Sat(width) * Sat(original_height); let use_width = nratio <= ratio; let intermediate = if use_width { - u64::from(original_height) * u64::from(width) / u64::from(original_width) + Sat(original_height) * Sat(checked!(width / original_width)?) } else { - u64::from(original_width) * u64::from(height) / u64::from(original_height) + Sat(original_width) * Sat(checked!(height / original_height)?) }; + if use_width { - if u32::try_from(intermediate).is_ok() { - (width, intermediate as u32) - } else { - ((u64::from(width) * u64::from(u32::MAX) / intermediate) as u32, u32::MAX) - } - } else if u32::try_from(intermediate).is_ok() { - (intermediate as u32, height) + (width, intermediate.0) } else { - (u32::MAX, (u64::from(height) * u64::from(u32::MAX) / intermediate) as u32) + (intermediate.0, height) } }; diff --git a/src/service/presence/mod.rs b/src/service/presence/mod.rs index 584f1a6d..f5400379 100644 --- a/src/service/presence/mod.rs +++ b/src/service/presence/mod.rs @@ -3,7 +3,7 @@ mod data; use std::{sync::Arc, time::Duration}; use async_trait::async_trait; -use conduit::{debug, error, utils, Error, Result}; +use conduit::{checked, debug, error, utils, Error, Result}; use data::Data; use futures_util::{stream::FuturesUnordered, StreamExt}; use ruma::{ @@ -79,12 +79,16 @@ pub struct Service { timer_receiver: Mutex>, handler_join: Mutex>>, timeout_remote_users: bool, + idle_timeout: u64, + offline_timeout: u64, } #[async_trait] impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let config = &args.server.config; + let idle_timeout_s = config.presence_idle_timeout_s; + let offline_timeout_s = config.presence_offline_timeout_s; let (timer_sender, timer_receiver) = loole::unbounded(); Ok(Arc::new(Self { db: Data::new(args.db), @@ -92,6 +96,8 @@ impl crate::Service for Service { timer_receiver: Mutex::new(timer_receiver), handler_join: Mutex::new(None), timeout_remote_users: config.presence_timeout_remote_users, + idle_timeout: checked!(idle_timeout_s * 1_000)?, + offline_timeout: checked!(offline_timeout_s * 1_000)?, })) } @@ -219,7 +225,7 @@ impl Service { loop { debug_assert!(!receiver.is_closed(), "channel error"); tokio::select! { - Some(user_id) = presence_timers.next() => process_presence_timer(&user_id)?, + Some(user_id) = presence_timers.next() => self.process_presence_timer(&user_id)?, event = receiver.recv_async() => match event { Err(_e) => return Ok(()), Ok((user_id, timeout)) => { @@ -230,6 +236,36 @@ impl Service { } } } + + fn process_presence_timer(&self, user_id: &OwnedUserId) -> Result<()> { + let mut presence_state = PresenceState::Offline; + let mut last_active_ago = None; + let mut status_msg = None; + + let presence_event = self.get_presence(user_id)?; + + if let Some(presence_event) = presence_event { + presence_state = presence_event.content.presence; + last_active_ago = presence_event.content.last_active_ago; + status_msg = presence_event.content.status_msg; + } + + let new_state = match (&presence_state, last_active_ago.map(u64::from)) { + (PresenceState::Online, Some(ago)) if ago >= self.idle_timeout => Some(PresenceState::Unavailable), + (PresenceState::Unavailable, Some(ago)) if ago >= self.offline_timeout => Some(PresenceState::Offline), + _ => None, + }; + + debug!( + "Processed presence timer for user '{user_id}': Old state = {presence_state}, New state = {new_state:?}" + ); + + if let Some(new_state) = new_state { + self.set_presence(user_id, &new_state, Some(false), last_active_ago, status_msg)?; + } + + Ok(()) + } } async fn presence_timer(user_id: OwnedUserId, timeout: Duration) -> OwnedUserId { @@ -237,36 +273,3 @@ async fn presence_timer(user_id: OwnedUserId, timeout: Duration) -> OwnedUserId user_id } - -fn process_presence_timer(user_id: &OwnedUserId) -> Result<()> { - let idle_timeout = services().globals.config.presence_idle_timeout_s * 1_000; - let offline_timeout = services().globals.config.presence_offline_timeout_s * 1_000; - - let mut presence_state = PresenceState::Offline; - let mut last_active_ago = None; - let mut status_msg = None; - - let presence_event = services().presence.get_presence(user_id)?; - - if let Some(presence_event) = presence_event { - presence_state = presence_event.content.presence; - last_active_ago = presence_event.content.last_active_ago; - status_msg = presence_event.content.status_msg; - } - - let new_state = match (&presence_state, last_active_ago.map(u64::from)) { - (PresenceState::Online, Some(ago)) if ago >= idle_timeout => Some(PresenceState::Unavailable), - (PresenceState::Unavailable, Some(ago)) if ago >= offline_timeout => Some(PresenceState::Offline), - _ => None, - }; - - debug!("Processed presence timer for user '{user_id}': Old state = {presence_state}, New state = {new_state:?}"); - - if let Some(new_state) = new_state { - services() - .presence - .set_presence(user_id, &new_state, Some(false), last_active_ago, status_msg)?; - } - - Ok(()) -} diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index ca04b1e5..18cdb70f 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -5,7 +5,7 @@ use std::{ sync::Arc, }; -use conduit::{debug, error, trace, warn, Error, Result}; +use conduit::{debug, error, trace, validated, warn, Error, Result}; use data::Data; use ruma::{api::client::error::ErrorKind, EventId, RoomId}; @@ -43,20 +43,20 @@ impl Service { #[tracing::instrument(skip_all, name = "auth_chain")] pub async fn get_auth_chain(&self, room_id: &RoomId, starting_events: &[&EventId]) -> Result> { - const NUM_BUCKETS: usize = 50; //TODO: change possible w/o disrupting db? + const NUM_BUCKETS: u64 = 50; //TODO: change possible w/o disrupting db? const BUCKET: BTreeSet<(u64, &EventId)> = BTreeSet::new(); let started = std::time::Instant::now(); - let mut buckets = [BUCKET; NUM_BUCKETS]; - for (i, short) in services() + let mut buckets = [BUCKET; NUM_BUCKETS as usize]; + for (i, &short) in services() .rooms .short .multi_get_or_create_shorteventid(starting_events)? .iter() .enumerate() { - let bucket = short % NUM_BUCKETS as u64; - buckets[bucket as usize].insert((*short, starting_events[i])); + let bucket = validated!(short % NUM_BUCKETS)?; + buckets[bucket as usize].insert((short, starting_events[i])); } debug!( diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 395b70f2..0f7919dd 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -191,7 +191,7 @@ impl Service { e.insert((Instant::now(), 1)); }, hash_map::Entry::Occupied(mut e) => { - *e.get_mut() = (Instant::now(), e.get().1 + 1); + *e.get_mut() = (Instant::now(), e.get().1.saturating_add(1)); }, }; }, @@ -1072,7 +1072,7 @@ impl Service { let mut todo_auth_events = vec![Arc::clone(id)]; let mut events_in_reverse_order = Vec::with_capacity(todo_auth_events.len()); let mut events_all = HashSet::with_capacity(todo_auth_events.len()); - let mut i = 0; + let mut i: u64 = 0; while let Some(next_id) = todo_auth_events.pop() { if let Some((time, tries)) = services() .globals @@ -1094,7 +1094,7 @@ impl Service { continue; } - i += 1; + i = i.saturating_add(1); if i % 100 == 0 { tokio::task::yield_now().await; } diff --git a/src/service/rooms/pdu_metadata/mod.rs b/src/service/rooms/pdu_metadata/mod.rs index e35c969d..cdb2fc29 100644 --- a/src/service/rooms/pdu_metadata/mod.rs +++ b/src/service/rooms/pdu_metadata/mod.rs @@ -205,7 +205,7 @@ impl Service { if let Ok(relations) = self.db.relations_until(user_id, room_id, target, until) { for relation in relations.flatten() { if stack_pdu.1 < max_depth { - stack.push((relation.clone(), stack_pdu.1 + 1)); + stack.push((relation.clone(), stack_pdu.1.saturating_add(1))); } pdus.push(relation); diff --git a/src/service/rooms/read_receipt/data.rs b/src/service/rooms/read_receipt/data.rs index 17acb0b3..06eaf655 100644 --- a/src/service/rooms/read_receipt/data.rs +++ b/src/service/rooms/read_receipt/data.rs @@ -76,10 +76,12 @@ impl Data { .iter_from(&first_possible_edu, false) .take_while(move |(k, _)| k.starts_with(&prefix2)) .map(move |(k, v)| { - let count = utils::u64_from_bytes(&k[prefix.len()..prefix.len() + size_of::()]) + let count_offset = prefix.len().saturating_add(size_of::()); + let count = utils::u64_from_bytes(&k[prefix.len()..count_offset]) .map_err(|_| Error::bad_database("Invalid readreceiptid count in db."))?; + let user_id_offset = count_offset.saturating_add(1); let user_id = UserId::parse( - utils::string_from_bytes(&k[prefix.len() + size_of::() + 1..]) + utils::string_from_bytes(&k[user_id_offset..]) .map_err(|_| Error::bad_database("Invalid readreceiptid userid bytes in db."))?, ) .map_err(|_| Error::bad_database("Invalid readreceiptid userid in db."))?; diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index bf6cc873..6924c50e 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -7,7 +7,7 @@ use std::{ sync::Arc, }; -use conduit::debug_info; +use conduit::{checked, debug_info}; use lru_cache::LruCache; use ruma::{ api::{ @@ -508,7 +508,8 @@ impl Service { } // We have reached the room after where we last left off - if parents.len() + 1 == short_room_ids.len() { + let parents_len = parents.len(); + if checked!(parents_len + 1)? == short_room_ids.len() { populate_results = true; } } diff --git a/src/service/rooms/state_compressor/data.rs b/src/service/rooms/state_compressor/data.rs index 61c7d6e6..33773001 100644 --- a/src/service/rooms/state_compressor/data.rs +++ b/src/service/rooms/state_compressor/data.rs @@ -1,6 +1,6 @@ use std::{collections::HashSet, mem::size_of, sync::Arc}; -use conduit::{utils, Error, Result}; +use conduit::{checked, utils, Error, Result}; use database::{Database, Map}; use super::CompressedStateEvent; @@ -38,11 +38,12 @@ impl Data { let mut added = HashSet::new(); let mut removed = HashSet::new(); - let mut i = size_of::(); - while let Some(v) = value.get(i..i + 2 * size_of::()) { + let stride = size_of::(); + let mut i = stride; + while let Some(v) = value.get(i..checked!(i + 2 * stride)?) { if add_mode && v.starts_with(&0_u64.to_be_bytes()) { add_mode = false; - i += size_of::(); + i = checked!(i + stride)?; continue; } if add_mode { @@ -50,7 +51,7 @@ impl Data { } else { removed.insert(v.try_into().expect("we checked the size above")); } - i += 2 * size_of::(); + i = checked!(i + 2 * stride)?; } Ok(StateDiff { diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index 97f3fb80..ca076fc9 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -7,7 +7,7 @@ use std::{ sync::{Arc, Mutex as StdMutex, Mutex}, }; -use conduit::{utils, Result}; +use conduit::{checked, utils, Result}; use data::Data; use lru_cache::LruCache; use ruma::{EventId, RoomId}; @@ -169,12 +169,14 @@ impl Service { statediffremoved: Arc>, diff_to_sibling: usize, mut parent_states: ParentStatesVec, ) -> Result<()> { - let diffsum = statediffnew.len() + statediffremoved.len(); + let statediffnew_len = statediffnew.len(); + let statediffremoved_len = statediffremoved.len(); + let diffsum = checked!(statediffnew_len + statediffremoved_len)?; if parent_states.len() > 3 { // Number of layers // To many layers, we have to go deeper - let parent = parent_states.pop().unwrap(); + let parent = parent_states.pop().expect("parent must have a state"); let mut parent_new = (*parent.2).clone(); let mut parent_removed = (*parent.3).clone(); @@ -226,10 +228,12 @@ impl Service { // 1. We add the current diff on top of the parent layer. // 2. We replace a layer above - let parent = parent_states.pop().unwrap(); - let parent_diff = parent.2.len() + parent.3.len(); + let parent = parent_states.pop().expect("parent must have a state"); + let parent_2_len = parent.2.len(); + let parent_3_len = parent.3.len(); + let parent_diff = checked!(parent_2_len + parent_3_len)?; - if diffsum * diffsum >= 2 * diff_to_sibling * parent_diff { + if checked!(diffsum * diffsum)? >= checked!(2 * diff_to_sibling * parent_diff)? { // Diff too big, we replace above layer(s) let mut parent_new = (*parent.2).clone(); let mut parent_removed = (*parent.3).clone(); diff --git a/src/service/rooms/threads/data.rs b/src/service/rooms/threads/data.rs index 29539847..c4a1a294 100644 --- a/src/service/rooms/threads/data.rs +++ b/src/service/rooms/threads/data.rs @@ -1,6 +1,6 @@ use std::{mem::size_of, sync::Arc}; -use conduit::{utils, Error, Result}; +use conduit::{checked, utils, Error, Result}; use database::{Database, Map}; use ruma::{api::client::threads::get_threads::v1::IncludeThreads, OwnedUserId, RoomId, UserId}; @@ -31,7 +31,7 @@ impl Data { .to_vec(); let mut current = prefix.clone(); - current.extend_from_slice(&(until - 1).to_be_bytes()); + current.extend_from_slice(&(checked!(until - 1)?).to_be_bytes()); Ok(Box::new( self.threadid_userids diff --git a/src/service/rooms/threads/mod.rs b/src/service/rooms/threads/mod.rs index f3cefe21..dd2686b0 100644 --- a/src/service/rooms/threads/mod.rs +++ b/src/service/rooms/threads/mod.rs @@ -64,7 +64,7 @@ impl Service { .and_then(|relations| serde_json::from_value::(relations.clone().into()).ok()) { // Thread already existed - relations.count += uint!(1); + relations.count = relations.count.saturating_add(uint!(1)); relations.latest_event = pdu.to_message_like_event(); let content = serde_json::to_value(relations).expect("to_value always works"); diff --git a/src/service/rooms/timeline/data.rs b/src/service/rooms/timeline/data.rs index 0d4d945e..ec975b99 100644 --- a/src/service/rooms/timeline/data.rs +++ b/src/service/rooms/timeline/data.rs @@ -4,7 +4,7 @@ use std::{ sync::{Arc, Mutex}, }; -use conduit::{error, utils, Error, Result}; +use conduit::{checked, error, utils, Error, Result}; use database::{Database, Map}; use ruma::{api::client::error::ErrorKind, CanonicalJsonObject, EventId, OwnedRoomId, OwnedUserId, RoomId, UserId}; @@ -281,10 +281,12 @@ impl Data { /// Returns the `count` of this pdu's id. pub(super) fn pdu_count(pdu_id: &[u8]) -> Result { - let last_u64 = utils::u64_from_bytes(&pdu_id[pdu_id.len() - size_of::()..]) + let stride = size_of::(); + let pdu_id_len = pdu_id.len(); + let last_u64 = utils::u64_from_bytes(&pdu_id[checked!(pdu_id_len - stride)?..]) .map_err(|_| Error::bad_database("PDU has invalid count bytes."))?; let second_last_u64 = - utils::u64_from_bytes(&pdu_id[pdu_id.len() - 2 * size_of::()..pdu_id.len() - size_of::()]); + utils::u64_from_bytes(&pdu_id[checked!(pdu_id_len - 2 * stride)?..checked!(pdu_id_len - stride)?]); if matches!(second_last_u64, Ok(0)) { Ok(PduCount::Backfilled(u64::MAX.saturating_sub(last_u64))) diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index ba987dbd..70f4423c 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -6,7 +6,7 @@ use std::{ sync::Arc, }; -use conduit::{debug, error, info, utils, utils::mutex_map, warn, Error, Result}; +use conduit::{debug, error, info, utils, utils::mutex_map, validated, warn, Error, Result}; use data::Data; use itertools::Itertools; use rand::prelude::SliceRandom; @@ -670,7 +670,7 @@ impl Service { .filter_map(|event_id| Some(self.get_pdu(event_id).ok()??.depth)) .max() .unwrap_or_else(|| uint!(0)) - + uint!(1); + .saturating_add(uint!(1)); let mut unsigned = unsigned.unwrap_or_default(); @@ -1240,10 +1240,11 @@ impl Service { let insert_lock = services().globals.roomid_mutex_insert.lock(&room_id).await; + let max = u64::MAX; let count = services().globals.next_count()?; let mut pdu_id = shortroomid.to_be_bytes().to_vec(); pdu_id.extend_from_slice(&0_u64.to_be_bytes()); - pdu_id.extend_from_slice(&(u64::MAX - count).to_be_bytes()); + pdu_id.extend_from_slice(&(validated!(max - count)?).to_be_bytes()); // Insert pdu self.db.prepend_backfill_pdu(&pdu_id, &event_id, &value)?; diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 0f4fa17a..0fb0d9dc 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -93,7 +93,7 @@ impl Service { statuses.entry(dest).and_modify(|e| { *e = match e { TransactionStatus::Running => TransactionStatus::Failed(1, Instant::now()), - TransactionStatus::Retrying(n) => TransactionStatus::Failed(*n + 1, Instant::now()), + TransactionStatus::Retrying(ref n) => TransactionStatus::Failed(n.saturating_add(1), Instant::now()), TransactionStatus::Failed(..) => panic!("Request that was not even running failed?!"), } }); diff --git a/src/service/users/data.rs b/src/service/users/data.rs index 3ba54a93..302eae9d 100644 --- a/src/service/users/data.rs +++ b/src/service/users/data.rs @@ -463,7 +463,8 @@ impl Data { .algorithm(), ) }) { - *counts.entry(algorithm?).or_default() += uint!(1); + let count: &mut UInt = counts.entry(algorithm?).or_default(); + *count = count.saturating_add(uint!(1)); } Ok(counts) @@ -814,7 +815,7 @@ impl Data { .map(|(key, _)| { Ok::<_, Error>(( key.clone(), - utils::u64_from_bytes(&key[key.len() - size_of::()..key.len()]) + utils::u64_from_bytes(&key[key.len().saturating_sub(size_of::())..key.len()]) .map_err(|_| Error::bad_database("ToDeviceId has invalid count bytes."))?, )) }) @@ -928,10 +929,12 @@ impl Data { /// Creates an OpenID token, which can be used to prove that a user has /// access to an account (primarily for integrations) pub(super) fn create_openid_token(&self, user_id: &UserId, token: &str) -> Result { - let expires_in = services().globals.config.openid_token_ttl; - let expires_at = utils::millis_since_unix_epoch().saturating_add(expires_in * 1000); + use std::num::Saturating as Sat; - let mut value = expires_at.to_be_bytes().to_vec(); + let expires_in = services().globals.config.openid_token_ttl; + let expires_at = Sat(utils::millis_since_unix_epoch()) + Sat(expires_in) * Sat(1000); + + let mut value = expires_at.0.to_be_bytes().to_vec(); value.extend_from_slice(user_id.as_bytes()); self.openidtoken_expiresatuserid From dcd7422c45a0d124bd6689e339760bd330facb2b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 7 Jul 2024 06:17:58 +0000 Subject: [PATCH 033/158] fix as conversions Signed-off-by: Jason Volk --- Cargo.toml | 2 +- src/api/client/space.rs | 2 +- src/api/client/sync.rs | 32 ++++++++++++--------- src/core/alloc/je.rs | 25 ++++++++++------ src/core/utils/math.rs | 35 +++++++++++++++++++++++ src/core/utils/time.rs | 4 ++- src/service/rooms/auth_chain/data.rs | 4 +-- src/service/rooms/auth_chain/mod.rs | 9 +++--- src/service/rooms/spaces/mod.rs | 19 ++++++------ src/service/rooms/state_accessor/mod.rs | 19 +++++++----- src/service/rooms/state_compressor/mod.rs | 7 ++--- 11 files changed, 107 insertions(+), 51 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 19ccbb00..bf4f71c7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -770,7 +770,7 @@ perf = "warn" #restriction = "warn" arithmetic_side_effects = "warn" -#as_conversions = "warn" # TODO +as_conversions = "warn" assertions_on_result_states = "warn" dbg_macro = "warn" default_union_representation = "warn" diff --git a/src/api/client/space.rs b/src/api/client/space.rs index e00171c3..0cf1b107 100644 --- a/src/api/client/space.rs +++ b/src/api/client/space.rs @@ -47,7 +47,7 @@ pub(crate) async fn get_hierarchy_route(body: Ruma) &body.room_id, limit.try_into().unwrap_or(10), key.map_or(vec![], |token| token.short_room_ids), - max_depth.try_into().unwrap_or(3), + max_depth.into(), body.suggested_only, ) .await diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index 2ea766a4..9ef1fab0 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -4,7 +4,11 @@ use std::{ time::Duration, }; -use conduit::PduCount; +use conduit::{ + error, + utils::math::{ruma_from_u64, ruma_from_usize, usize_from_ruma, usize_from_u64_truncated}, + PduCount, +}; use ruma::{ api::client::{ filter::{FilterDefinition, LazyLoadOptions}, @@ -27,7 +31,7 @@ use ruma::{ serde::Raw, uint, DeviceId, EventId, OwnedUserId, RoomId, UInt, UserId, }; -use tracing::{error, Instrument as _, Span}; +use tracing::{Instrument as _, Span}; use crate::{service::pdu::EventHash, services, utils, Error, PduEvent, Result, Ruma, RumaResponse}; @@ -975,8 +979,8 @@ async fn load_joined_room( }, summary: RoomSummary { heroes, - joined_member_count: joined_member_count.map(|n| (n as u32).into()), - invited_member_count: invited_member_count.map(|n| (n as u32).into()), + joined_member_count: joined_member_count.map(ruma_from_u64), + invited_member_count: invited_member_count.map(ruma_from_u64), }, unread_notifications: UnreadNotificationsCount { highlight_count, @@ -1026,7 +1030,7 @@ fn load_timeline( // Take the last events for the timeline timeline_pdus = non_timeline_pdus .by_ref() - .take(limit as usize) + .take(usize_from_u64_truncated(limit)) .collect::>() .into_iter() .rev() @@ -1300,7 +1304,7 @@ pub(crate) async fn sync_events_v4_route( r.0, UInt::try_from(all_joined_rooms.len().saturating_sub(1)).unwrap_or(UInt::MAX), ); - let room_ids = all_joined_rooms[(u64::from(r.0) as usize)..=(u64::from(r.1) as usize)].to_vec(); + let room_ids = all_joined_rooms[usize_from_ruma(r.0)..=usize_from_ruma(r.1)].to_vec(); new_known_rooms.extend(room_ids.iter().cloned()); for room_id in &room_ids { let todo_room = todo_rooms @@ -1333,7 +1337,7 @@ pub(crate) async fn sync_events_v4_route( } }) .collect(), - count: UInt::from(all_joined_rooms.len() as u32), + count: ruma_from_usize(all_joined_rooms.len()), }, ); @@ -1529,20 +1533,22 @@ pub(crate) async fn sync_events_v4_route( prev_batch, limited, joined_count: Some( - (services() + services() .rooms .state_cache .room_joined_count(room_id)? - .unwrap_or(0) as u32) - .into(), + .unwrap_or(0) + .try_into() + .unwrap_or_else(|_| uint!(0)), ), invited_count: Some( - (services() + services() .rooms .state_cache .room_invited_count(room_id)? - .unwrap_or(0) as u32) - .into(), + .unwrap_or(0) + .try_into() + .unwrap_or_else(|_| uint!(0)), ), num_live: None, // Count events in timeline greater than global sync counter timestamp: None, diff --git a/src/core/alloc/je.rs b/src/core/alloc/je.rs index 966cbde3..5e3c361f 100644 --- a/src/core/alloc/je.rs +++ b/src/core/alloc/je.rs @@ -12,15 +12,24 @@ static JEMALLOC: jemalloc::Jemalloc = jemalloc::Jemalloc; #[must_use] pub fn memory_usage() -> String { use mallctl::stats; - let allocated = stats::allocated::read().unwrap_or_default() as f64 / 1024.0 / 1024.0; - let active = stats::active::read().unwrap_or_default() as f64 / 1024.0 / 1024.0; - let mapped = stats::mapped::read().unwrap_or_default() as f64 / 1024.0 / 1024.0; - let metadata = stats::metadata::read().unwrap_or_default() as f64 / 1024.0 / 1024.0; - let resident = stats::resident::read().unwrap_or_default() as f64 / 1024.0 / 1024.0; - let retained = stats::retained::read().unwrap_or_default() as f64 / 1024.0 / 1024.0; + + let mibs = |input: Result| { + let input = input.unwrap_or_default(); + let kibs = input / 1024; + let kibs = u32::try_from(kibs).unwrap_or_default(); + let kibs = f64::from(kibs); + kibs / 1024.0 + }; + + let allocated = mibs(stats::allocated::read()); + let active = mibs(stats::active::read()); + let mapped = mibs(stats::mapped::read()); + let metadata = mibs(stats::metadata::read()); + let resident = mibs(stats::resident::read()); + let retained = mibs(stats::retained::read()); format!( - "allocated: {allocated:.2} MiB\n active: {active:.2} MiB\n mapped: {mapped:.2} MiB\n metadata: {metadata:.2} \ - MiB\n resident: {resident:.2} MiB\n retained: {retained:.2} MiB\n " + "allocated: {allocated:.2} MiB\nactive: {active:.2} MiB\nmapped: {mapped:.2} MiB\nmetadata: {metadata:.2} \ + MiB\nresident: {resident:.2} MiB\nretained: {retained:.2} MiB\n" ) } diff --git a/src/core/utils/math.rs b/src/core/utils/math.rs index f00b9055..3dc7e27f 100644 --- a/src/core/utils/math.rs +++ b/src/core/utils/math.rs @@ -2,6 +2,8 @@ use std::{cmp, time::Duration}; pub use checked_ops::checked_ops; +use crate::{Error, Result}; + /// Checked arithmetic expression. Returns a Result #[macro_export] macro_rules! checked { @@ -50,3 +52,36 @@ pub fn continue_exponential_backoff(min: Duration, max: Duration, elapsed: Durat let min = cmp::min(min, max); elapsed < min } + +#[inline] +#[allow(clippy::as_conversions)] +pub fn usize_from_f64(val: f64) -> Result { + if val < 0.0 { + return Err(Error::Arithmetic("Converting negative float to unsigned integer")); + } + + Ok(val as usize) +} + +#[inline] +#[must_use] +pub fn usize_from_ruma(val: ruma::UInt) -> usize { + usize::try_from(val).expect("failed conversion from ruma::UInt to usize") +} + +#[inline] +#[must_use] +pub fn ruma_from_u64(val: u64) -> ruma::UInt { + ruma::UInt::try_from(val).expect("failed conversion from u64 to ruma::UInt") +} + +#[inline] +#[must_use] +pub fn ruma_from_usize(val: usize) -> ruma::UInt { + ruma::UInt::try_from(val).expect("failed conversion from usize to ruma::UInt") +} + +#[inline] +#[must_use] +#[allow(clippy::as_conversions)] +pub fn usize_from_u64_truncated(val: u64) -> usize { val as usize } diff --git a/src/core/utils/time.rs b/src/core/utils/time.rs index 2fe90b31..bf6192b2 100644 --- a/src/core/utils/time.rs +++ b/src/core/utils/time.rs @@ -28,11 +28,12 @@ pub fn format(ts: SystemTime, str: &str) -> String { } #[must_use] +#[allow(clippy::as_conversions)] pub fn pretty(d: Duration) -> String { use Unit::*; let fmt = |w, f, u| format!("{w}.{f} {u}"); - let gen64 = |w, f, u| fmt(w, (f * 100.0) as u64, u); + let gen64 = |w, f, u| fmt(w, (f * 100.0) as u32, u); let gen128 = |w, f, u| gen64(u64::try_from(w).expect("u128 to u64"), f, u); match whole_and_frac(d) { (Days(whole), frac) => gen64(whole, frac, "days"), @@ -49,6 +50,7 @@ pub fn pretty(d: Duration) -> String { /// part is the largest Unit containing a non-zero value, the frac part is a /// rational remainder left over. #[must_use] +#[allow(clippy::as_conversions)] pub fn whole_and_frac(d: Duration) -> (Unit, f64) { use Unit::*; diff --git a/src/service/rooms/auth_chain/data.rs b/src/service/rooms/auth_chain/data.rs index a5771f4a..5efb36c2 100644 --- a/src/service/rooms/auth_chain/data.rs +++ b/src/service/rooms/auth_chain/data.rs @@ -3,7 +3,7 @@ use std::{ sync::{Arc, Mutex}, }; -use conduit::{utils, Result, Server}; +use conduit::{utils, utils::math::usize_from_f64, Result, Server}; use database::{Database, Map}; use lru_cache::LruCache; @@ -16,7 +16,7 @@ impl Data { pub(super) fn new(server: &Arc, db: &Arc) -> Self { let config = &server.config; let cache_size = f64::from(config.auth_chain_cache_capacity); - let cache_size = (cache_size * config.conduit_cache_capacity_modifier) as usize; + let cache_size = usize_from_f64(cache_size * config.conduit_cache_capacity_modifier).expect("valid cache size"); Self { shorteventid_authchain: db["shorteventid_authchain"].clone(), auth_chain_cache: Mutex::new(LruCache::new(cache_size)), diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index 18cdb70f..7b2a4f01 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -43,11 +43,11 @@ impl Service { #[tracing::instrument(skip_all, name = "auth_chain")] pub async fn get_auth_chain(&self, room_id: &RoomId, starting_events: &[&EventId]) -> Result> { - const NUM_BUCKETS: u64 = 50; //TODO: change possible w/o disrupting db? + const NUM_BUCKETS: usize = 50; //TODO: change possible w/o disrupting db? const BUCKET: BTreeSet<(u64, &EventId)> = BTreeSet::new(); let started = std::time::Instant::now(); - let mut buckets = [BUCKET; NUM_BUCKETS as usize]; + let mut buckets = [BUCKET; NUM_BUCKETS]; for (i, &short) in services() .rooms .short @@ -55,8 +55,9 @@ impl Service { .iter() .enumerate() { - let bucket = validated!(short % NUM_BUCKETS)?; - buckets[bucket as usize].insert((short, starting_events[i])); + let bucket: usize = short.try_into()?; + let bucket: usize = validated!(bucket % NUM_BUCKETS)?; + buckets[bucket].insert((short, starting_events[i])); } debug!( diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index 6924c50e..03d0d43f 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -7,7 +7,7 @@ use std::{ sync::Arc, }; -use conduit::{checked, debug_info}; +use conduit::{checked, debug_info, utils::math::usize_from_f64}; use lru_cache::LruCache; use ruma::{ api::{ @@ -161,11 +161,10 @@ impl From for SpaceHierarchyRoomsChunk { impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let config = &args.server.config; + let cache_size = f64::from(config.roomid_spacehierarchy_cache_capacity); + let cache_size = cache_size * config.conduit_cache_capacity_modifier; Ok(Arc::new(Self { - roomid_spacehierarchy_cache: Mutex::new(LruCache::new( - (f64::from(config.roomid_spacehierarchy_cache_capacity) * config.conduit_cache_capacity_modifier) - as usize, - )), + roomid_spacehierarchy_cache: Mutex::new(LruCache::new(usize_from_f64(cache_size)?)), })) } @@ -447,7 +446,7 @@ impl Service { } pub async fn get_client_hierarchy( - &self, sender_user: &UserId, room_id: &RoomId, limit: usize, short_room_ids: Vec, max_depth: usize, + &self, sender_user: &UserId, room_id: &RoomId, limit: usize, short_room_ids: Vec, max_depth: u64, suggested_only: bool, ) -> Result { let mut parents = VecDeque::new(); @@ -514,7 +513,8 @@ impl Service { } } - if !children.is_empty() && parents.len() < max_depth { + let parents_len: u64 = parents.len().try_into()?; + if !children.is_empty() && parents_len < max_depth { parents.push_back(current_room.clone()); stack.push(children); } @@ -549,9 +549,8 @@ impl Service { Some( PaginationToken { short_room_ids, - limit: UInt::new(max_depth as u64).expect("When sent in request it must have been valid UInt"), - max_depth: UInt::new(max_depth as u64) - .expect("When sent in request it must have been valid UInt"), + limit: UInt::new(max_depth).expect("When sent in request it must have been valid UInt"), + max_depth: UInt::new(max_depth).expect("When sent in request it must have been valid UInt"), suggested_only, } .to_string(), diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 7fe61134..e8554500 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -6,7 +6,11 @@ use std::{ sync::{Arc, Mutex as StdMutex, Mutex}, }; -use conduit::{error, utils::mutex_map, warn, Error, Result}; +use conduit::{ + error, + utils::{math::usize_from_f64, mutex_map}, + warn, Error, Result, +}; use data::Data; use lru_cache::LruCache; use ruma::{ @@ -44,14 +48,15 @@ pub struct Service { impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let config = &args.server.config; + let server_visibility_cache_capacity = + f64::from(config.server_visibility_cache_capacity) * config.conduit_cache_capacity_modifier; + let user_visibility_cache_capacity = + f64::from(config.user_visibility_cache_capacity) * config.conduit_cache_capacity_modifier; + Ok(Arc::new(Self { db: Data::new(args.db), - server_visibility_cache: StdMutex::new(LruCache::new( - (f64::from(config.server_visibility_cache_capacity) * config.conduit_cache_capacity_modifier) as usize, - )), - user_visibility_cache: StdMutex::new(LruCache::new( - (f64::from(config.user_visibility_cache_capacity) * config.conduit_cache_capacity_modifier) as usize, - )), + server_visibility_cache: StdMutex::new(LruCache::new(usize_from_f64(server_visibility_cache_capacity)?)), + user_visibility_cache: StdMutex::new(LruCache::new(usize_from_f64(user_visibility_cache_capacity)?)), })) } diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index ca076fc9..197efbea 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -7,7 +7,7 @@ use std::{ sync::{Arc, Mutex as StdMutex, Mutex}, }; -use conduit::{checked, utils, Result}; +use conduit::{checked, utils, utils::math::usize_from_f64, Result}; use data::Data; use lru_cache::LruCache; use ruma::{EventId, RoomId}; @@ -55,11 +55,10 @@ pub struct Service { impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let config = &args.server.config; + let cache_capacity = f64::from(config.stateinfo_cache_capacity) * config.conduit_cache_capacity_modifier; Ok(Arc::new(Self { db: Data::new(args.db), - stateinfo_cache: StdMutex::new(LruCache::new( - (f64::from(config.stateinfo_cache_capacity) * config.conduit_cache_capacity_modifier) as usize, - )), + stateinfo_cache: StdMutex::new(LruCache::new(usize_from_f64(cache_capacity)?)), })) } From dfd13780dff7a144c46c16be228d10a9c41da2ad Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 7 Jul 2024 07:14:09 +0000 Subject: [PATCH 034/158] mitigate additional cast lints Signed-off-by: Jason Volk --- Cargo.toml | 3 --- src/core/utils/math.rs | 5 +++-- src/core/utils/time.rs | 6 +++--- src/database/engine.rs | 27 ++++++++++++--------------- 4 files changed, 18 insertions(+), 23 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index bf4f71c7..af9c719e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -740,9 +740,6 @@ significant_drop_tightening = { level = "allow", priority = 1 } # TODO pedantic = "warn" ## some sadness -cast_possible_truncation = { level = "allow", priority = 1 } -cast_precision_loss = { level = "allow", priority = 1 } -cast_sign_loss = { level = "allow", priority = 1 } doc_markdown = { level = "allow", priority = 1 } enum_glob_use = { level = "allow", priority = 1 } error_impl_error = { level = "allow", priority = 1 } diff --git a/src/core/utils/math.rs b/src/core/utils/math.rs index 3dc7e27f..155721e7 100644 --- a/src/core/utils/math.rs +++ b/src/core/utils/math.rs @@ -60,7 +60,8 @@ pub fn usize_from_f64(val: f64) -> Result { return Err(Error::Arithmetic("Converting negative float to unsigned integer")); } - Ok(val as usize) + //SAFETY: + Ok(unsafe { val.to_int_unchecked::() }) } #[inline] @@ -83,5 +84,5 @@ pub fn ruma_from_usize(val: usize) -> ruma::UInt { #[inline] #[must_use] -#[allow(clippy::as_conversions)] +#[allow(clippy::as_conversions, clippy::cast_possible_truncation)] pub fn usize_from_u64_truncated(val: u64) -> usize { val as usize } diff --git a/src/core/utils/time.rs b/src/core/utils/time.rs index bf6192b2..9a31632e 100644 --- a/src/core/utils/time.rs +++ b/src/core/utils/time.rs @@ -2,7 +2,7 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; #[inline] #[must_use] -#[allow(clippy::as_conversions)] +#[allow(clippy::as_conversions, clippy::cast_possible_truncation)] pub fn now_millis() -> u64 { UNIX_EPOCH .elapsed() @@ -28,7 +28,7 @@ pub fn format(ts: SystemTime, str: &str) -> String { } #[must_use] -#[allow(clippy::as_conversions)] +#[allow(clippy::as_conversions, clippy::cast_possible_truncation, clippy::cast_sign_loss)] pub fn pretty(d: Duration) -> String { use Unit::*; @@ -50,7 +50,7 @@ pub fn pretty(d: Duration) -> String { /// part is the largest Unit containing a non-zero value, the frac part is a /// rational remainder left over. #[must_use] -#[allow(clippy::as_conversions)] +#[allow(clippy::as_conversions, clippy::cast_precision_loss)] pub fn whole_and_frac(d: Duration) -> (Unit, f64) { use Unit::*; diff --git a/src/database/engine.rs b/src/database/engine.rs index 7c9522e1..53e39abc 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -134,23 +134,21 @@ impl Engine { .fetch_sub(1, std::sync::atomic::Ordering::Relaxed); } - #[allow(clippy::as_conversions, clippy::cast_sign_loss, clippy::cast_possible_truncation)] pub fn memory_usage(&self) -> Result { let mut res = String::new(); let stats = get_memory_usage_stats(Some(&[&self.db]), Some(&[&self.row_cache])).or_else(or_else)?; + let mibs = |input| f64::from(u32::try_from(input / 1024).unwrap_or(0)) / 1024.0; writeln!( res, "Memory buffers: {:.2} MiB\nPending write: {:.2} MiB\nTable readers: {:.2} MiB\nRow cache: {:.2} MiB", - stats.mem_table_total as f64 / 1024.0 / 1024.0, - stats.mem_table_unflushed as f64 / 1024.0 / 1024.0, - stats.mem_table_readers_total as f64 / 1024.0 / 1024.0, - self.row_cache.get_usage() as f64 / 1024.0 / 1024.0, - ) - .expect("should be able to write to string buffer"); + mibs(stats.mem_table_total), + mibs(stats.mem_table_unflushed), + mibs(stats.mem_table_readers_total), + mibs(u64::try_from(self.row_cache.get_usage())?), + )?; for (name, cache) in &*self.col_cache.read().expect("locked") { - writeln!(res, "{} cache: {:.2} MiB", name, cache.get_usage() as f64 / 1024.0 / 1024.0,) - .expect("should be able to write to string buffer"); + writeln!(res, "{} cache: {:.2} MiB", name, mibs(u64::try_from(cache.get_usage())?))?; } Ok(res) @@ -214,8 +212,7 @@ impl Engine { rfc2822_from_seconds(info.timestamp), info.size, info.num_files, - ) - .expect("should be able to write to string buffer"); + )?; } Ok(res) @@ -226,16 +223,16 @@ impl Engine { Err(e) => Ok(String::from(e)), Ok(files) => { let mut res = String::new(); - writeln!(res, "| lev | sst | keys | dels | size | column |").expect("written to string buffer"); - writeln!(res, "| ---: | :--- | ---: | ---: | ---: | :--- |").expect("written to string buffer"); + writeln!(res, "| lev | sst | keys | dels | size | column |")?; + writeln!(res, "| ---: | :--- | ---: | ---: | ---: | :--- |")?; for file in files { writeln!( res, "| {} | {:<13} | {:7}+ | {:4}- | {:9} | {} |", file.level, file.name, file.num_entries, file.num_deletions, file.size, file.column_family_name, - ) - .expect("should be able to writeln to string buffer"); + )?; } + Ok(res) }, } From 5722c4ae396368d0b7c696b7106bbcf4b0f213a4 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 7 Jul 2024 07:39:18 +0000 Subject: [PATCH 035/158] fix needless collect Signed-off-by: Jason Volk --- Cargo.toml | 1 - src/admin/room/room_moderation_commands.rs | 20 +++++--------------- src/api/client/membership.rs | 3 +-- src/service/rooms/state_cache/data.rs | 4 +++- src/service/rooms/state_cache/mod.rs | 2 +- 5 files changed, 10 insertions(+), 20 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index af9c719e..f6149086 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -730,7 +730,6 @@ nursery = "warn" ## some sadness missing_const_for_fn = { level = "allow", priority = 1 } # TODO -needless_collect = { level = "allow", priority = 1 } # TODO option_if_let_else = { level = "allow", priority = 1 } # TODO redundant_pub_crate = { level = "allow", priority = 1 } # TODO significant_drop_in_scrutinee = { level = "allow", priority = 1 } # TODO diff --git a/src/admin/room/room_moderation_commands.rs b/src/admin/room/room_moderation_commands.rs index 92e3de47..30c30c6e 100644 --- a/src/admin/room/room_moderation_commands.rs +++ b/src/admin/room/room_moderation_commands.rs @@ -1,7 +1,5 @@ use api::client::leave_room; -use ruma::{ - events::room::message::RoomMessageEventContent, OwnedRoomId, OwnedUserId, RoomAliasId, RoomId, RoomOrAliasId, -}; +use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomId, RoomAliasId, RoomId, RoomOrAliasId}; use tracing::{debug, error, info, warn}; use super::{super::Service, RoomModerationCommand}; @@ -124,9 +122,7 @@ async fn ban_room( .is_admin(local_user) .unwrap_or(true)) }) - }) - .collect::>() - { + }) { debug!( "Attempting leave for user {} in room {} (forced, ignoring all errors, evicting admins too)", &local_user, &room_id @@ -153,9 +149,7 @@ async fn ban_room( .is_admin(local_user) .unwrap_or(false)) }) - }) - .collect::>() - { + }) { debug!("Attempting leave for user {} in room {}", &local_user, &room_id); if let Err(e) = leave_room(&local_user, &room_id, None).await { error!( @@ -335,9 +329,7 @@ async fn ban_list_of_rooms(body: Vec<&str>, force: bool, disable_federation: boo .is_admin(local_user) .unwrap_or(true)) }) - }) - .collect::>() - { + }) { debug!( "Attempting leave for user {} in room {} (forced, ignoring all errors, evicting admins too)", &local_user, room_id @@ -364,9 +356,7 @@ async fn ban_list_of_rooms(body: Vec<&str>, force: bool, disable_federation: boo .is_admin(local_user) .unwrap_or(false)) }) - }) - .collect::>() - { + }) { debug!("Attempting leave for user {} in room {}", &local_user, &room_id); if let Err(e) = leave_room(&local_user, &room_id, None).await { error!( diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index e2ba4c9a..0792ed89 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -1682,8 +1682,7 @@ async fn remote_leave_room(user_id: &UserId, room_id: &RoomId) -> Result<()> { .filter_map(|event: serde_json::Value| event.get("sender").cloned()) .filter_map(|sender| sender.as_str().map(ToOwned::to_owned)) .filter_map(|sender| UserId::parse(sender).ok()) - .map(|user| user.server_name().to_owned()) - .collect::>(), + .map(|user| user.server_name().to_owned()), ); debug!("servers in remote_leave_room: {servers:?}"); diff --git a/src/service/rooms/state_cache/data.rs b/src/service/rooms/state_cache/data.rs index f79ee678..0120d5a6 100644 --- a/src/service/rooms/state_cache/data.rs +++ b/src/service/rooms/state_cache/data.rs @@ -319,7 +319,9 @@ impl Data { /// Returns an iterator of all joined members of a room. #[tracing::instrument(skip(self))] - pub(super) fn room_members<'a>(&'a self, room_id: &RoomId) -> Box> + 'a> { + pub(super) fn room_members<'a>( + &'a self, room_id: &RoomId, + ) -> Box> + Send + 'a> { let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xFF); diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index 45e05c29..a0910c7c 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -292,7 +292,7 @@ impl Service { /// Returns an iterator over all joined members of a room. #[tracing::instrument(skip(self))] - pub fn room_members(&self, room_id: &RoomId) -> impl Iterator> + '_ { + pub fn room_members(&self, room_id: &RoomId) -> impl Iterator> + Send + '_ { self.db.room_members(room_id) } From 56a1b0e7615f83fde2b7a6fc909f835c5e0f7ee8 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 7 Jul 2024 11:20:03 +0000 Subject: [PATCH 036/158] restrict untripped clippies Signed-off-by: Jason Volk --- Cargo.toml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f6149086..dad0adeb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -741,17 +741,12 @@ pedantic = "warn" ## some sadness doc_markdown = { level = "allow", priority = 1 } enum_glob_use = { level = "allow", priority = 1 } -error_impl_error = { level = "allow", priority = 1 } -expect_used = { level = "allow", priority = 1 } if_not_else = { level = "allow", priority = 1 } if_then_some_else_none = { level = "allow", priority = 1 } -implicit_return = { level = "allow", priority = 1 } inline_always = { level = "allow", priority = 1 } -map_err_ignore = { level = "allow", priority = 1 } missing_docs_in_private_items = { level = "allow", priority = 1 } missing_errors_doc = { level = "allow", priority = 1 } missing_panics_doc = { level = "allow", priority = 1 } -mod_module_files = { level = "allow", priority = 1 } module_name_repetitions = { level = "allow", priority = 1 } no_effect_underscore_binding = { level = "allow", priority = 1 } similar_names = { level = "allow", priority = 1 } @@ -765,8 +760,10 @@ perf = "warn" ################### #restriction = "warn" +allow_attributes = "warn" arithmetic_side_effects = "warn" as_conversions = "warn" +as_underscore = "warn" assertions_on_result_states = "warn" dbg_macro = "warn" default_union_representation = "warn" @@ -804,6 +801,7 @@ unnecessary_safety_doc = "warn" unnecessary_self_imports = "warn" unneeded_field_pattern = "warn" unseparated_literal_suffix = "warn" +#unwrap_used = "warn" # TODO verbose_file_reads = "warn" ################### From 5af880e4f4b5861cc484c628cd89a5c44d181c95 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 7 Jul 2024 20:44:40 +0000 Subject: [PATCH 037/158] fix reference count on punned ColumnFamily Arc Signed-off-by: Jason Volk --- src/database/database.rs | 2 +- src/database/map.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/database/database.rs b/src/database/database.rs index 06a2d88f..44bb655c 100644 --- a/src/database/database.rs +++ b/src/database/database.rs @@ -6,7 +6,7 @@ use crate::{cork::Cork, maps, maps::Maps, Engine, Map}; pub struct Database { pub db: Arc, - pub map: Maps, + map: Maps, } impl Database { diff --git a/src/database/map.rs b/src/database/map.rs index 0b007307..1b35a72a 100644 --- a/src/database/map.rs +++ b/src/database/map.rs @@ -233,7 +233,7 @@ fn open(db: &Arc, name: &str) -> Result> { // closing the database (dropping `Engine`). Since `Arc` is a sibling // member along with this handle in `Map`, that is prevented. Ok(unsafe { - Arc::decrement_strong_count(cf_ptr); + Arc::increment_strong_count(cf_ptr); Arc::from_raw(cf_ptr) }) } From 113a27c1d5b4b3b373577c0124016da00eb5879c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 7 Jul 2024 19:59:04 +0000 Subject: [PATCH 038/158] use usize for max_request_size config item Signed-off-by: Jason Volk --- src/api/client/media.rs | 4 ++-- src/api/router/request.rs | 7 +------ src/core/config/mod.rs | 4 ++-- src/router/layers.rs | 10 +--------- src/service/globals/mod.rs | 2 -- 5 files changed, 6 insertions(+), 21 deletions(-) diff --git a/src/api/client/media.rs b/src/api/client/media.rs index cfb10c3d..44978caf 100644 --- a/src/api/client/media.rs +++ b/src/api/client/media.rs @@ -2,6 +2,7 @@ use std::{io::Cursor, sync::Arc, time::Duration}; +use conduit::{debug, error, utils::math::ruma_from_usize, warn}; use image::io::Reader as ImgReader; use ipaddress::IPAddress; use reqwest::Url; @@ -12,7 +13,6 @@ use ruma::api::client::{ get_media_preview, }, }; -use tracing::{debug, error, warn}; use webpage::HTML; use crate::{ @@ -44,7 +44,7 @@ pub(crate) async fn get_media_config_route( _body: Ruma, ) -> Result { Ok(get_media_config::v3::Response { - upload_size: services().globals.max_request_size().into(), + upload_size: ruma_from_usize(services().globals.config.max_request_size), }) } diff --git a/src/api/router/request.rs b/src/api/router/request.rs index 59639eaa..56c76619 100644 --- a/src/api/router/request.rs +++ b/src/api/router/request.rs @@ -29,12 +29,7 @@ pub(super) async fn from(request: hyper::Request) -> Result u64 { 10 } fn default_ip_lookup_strategy() -> u8 { 5 } -fn default_max_request_size() -> u32 { +fn default_max_request_size() -> usize { 20 * 1024 * 1024 // Default to 20 MB } diff --git a/src/router/layers.rs b/src/router/layers.rs index 4fe35164..db664b38 100644 --- a/src/router/layers.rs +++ b/src/router/layers.rs @@ -138,15 +138,7 @@ fn cors_layer(_server: &Server) -> CorsLayer { .max_age(Duration::from_secs(86400)) } -fn body_limit_layer(server: &Server) -> DefaultBodyLimit { - DefaultBodyLimit::max( - server - .config - .max_request_size - .try_into() - .expect("failed to convert max request size"), - ) -} +fn body_limit_layer(server: &Server) -> DefaultBodyLimit { DefaultBodyLimit::max(server.config.max_request_size) } #[allow(clippy::needless_pass_by_value)] #[tracing::instrument(skip_all)] diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 3c0e66c0..f28299b2 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -201,8 +201,6 @@ impl Service { pub fn server_name(&self) -> &ServerName { self.config.server_name.as_ref() } - pub fn max_request_size(&self) -> u32 { self.config.max_request_size } - pub fn max_fetch_prev_events(&self) -> u16 { self.config.max_fetch_prev_events } pub fn allow_registration(&self) -> bool { self.config.allow_registration } From 59c406230507be37482bdc457ddb2f2a630f3a20 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 7 Jul 2024 19:03:15 +0000 Subject: [PATCH 039/158] set trivial/leaf spans to debug level Signed-off-by: Jason Volk --- src/core/pdu/mod.rs | 20 +++---- src/core/utils/content_disposition.rs | 2 +- src/core/utils/hash/sha256.rs | 2 +- src/router/request.rs | 2 +- src/service/account_data/mod.rs | 2 +- src/service/globals/data.rs | 2 +- src/service/globals/mod.rs | 4 +- src/service/pusher/mod.rs | 2 +- src/service/rooms/alias/mod.rs | 6 +- src/service/rooms/auth_chain/mod.rs | 4 +- src/service/rooms/directory/mod.rs | 8 +-- .../rooms/event_handler/signing_keys.rs | 2 +- src/service/rooms/lazy_loading/mod.rs | 8 +-- src/service/rooms/outlier/mod.rs | 2 +- src/service/rooms/pdu_metadata/mod.rs | 10 ++-- src/service/rooms/read_receipt/mod.rs | 6 +- src/service/rooms/search/mod.rs | 6 +- src/service/rooms/state/mod.rs | 12 ++-- src/service/rooms/state_accessor/mod.rs | 10 ++-- src/service/rooms/state_cache/data.rs | 48 +++++++-------- src/service/rooms/state_cache/mod.rs | 58 +++++++++---------- src/service/rooms/state_compressor/mod.rs | 4 +- src/service/rooms/timeline/mod.rs | 12 ++-- src/service/sending/data.rs | 2 +- src/service/sending/mod.rs | 22 +++---- 25 files changed, 128 insertions(+), 128 deletions(-) diff --git a/src/core/pdu/mod.rs b/src/core/pdu/mod.rs index c7d93608..0c54812e 100644 --- a/src/core/pdu/mod.rs +++ b/src/core/pdu/mod.rs @@ -64,7 +64,7 @@ pub struct PduEvent { } impl PduEvent { - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn redact(&mut self, room_version_id: RoomVersionId, reason: &Self) -> crate::Result<()> { self.unsigned = None; @@ -158,7 +158,7 @@ impl PduEvent { (self.redacts.clone(), self.content.clone()) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn to_sync_room_event(&self) -> Raw { let (redacts, content) = self.copy_redacts(); let mut json = json!({ @@ -183,7 +183,7 @@ impl PduEvent { } /// This only works for events that are also AnyRoomEvents. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn to_any_event(&self) -> Raw { let (redacts, content) = self.copy_redacts(); let mut json = json!({ @@ -208,7 +208,7 @@ impl PduEvent { serde_json::from_value(json).expect("Raw::from_value always works") } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn to_room_event(&self) -> Raw { let (redacts, content) = self.copy_redacts(); let mut json = json!({ @@ -233,7 +233,7 @@ impl PduEvent { serde_json::from_value(json).expect("Raw::from_value always works") } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn to_message_like_event(&self) -> Raw { let (redacts, content) = self.copy_redacts(); let mut json = json!({ @@ -258,7 +258,7 @@ impl PduEvent { serde_json::from_value(json).expect("Raw::from_value always works") } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn to_state_event(&self) -> Raw { let mut json = json!({ "content": self.content, @@ -277,7 +277,7 @@ impl PduEvent { serde_json::from_value(json).expect("Raw::from_value always works") } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn to_sync_state_event(&self) -> Raw { let mut json = json!({ "content": self.content, @@ -295,7 +295,7 @@ impl PduEvent { serde_json::from_value(json).expect("Raw::from_value always works") } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn to_stripped_state_event(&self) -> Raw { let json = json!({ "content": self.content, @@ -307,7 +307,7 @@ impl PduEvent { serde_json::from_value(json).expect("Raw::from_value always works") } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn to_stripped_spacechild_state_event(&self) -> Raw { let json = json!({ "content": self.content, @@ -320,7 +320,7 @@ impl PduEvent { serde_json::from_value(json).expect("Raw::from_value always works") } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn to_member_event(&self) -> Raw> { let mut json = json!({ "content": self.content, diff --git a/src/core/utils/content_disposition.rs b/src/core/utils/content_disposition.rs index 1c2b066d..be17a731 100644 --- a/src/core/utils/content_disposition.rs +++ b/src/core/utils/content_disposition.rs @@ -66,7 +66,7 @@ pub fn content_disposition_type(content_type: &Option) -> &'static str { /// sanitises the file name for the Content-Disposition using /// `sanitize_filename` crate -#[tracing::instrument] +#[tracing::instrument(level = "debug")] pub fn sanitise_filename(filename: String) -> String { let options = sanitize_filename::Options { truncate: false, diff --git a/src/core/utils/hash/sha256.rs b/src/core/utils/hash/sha256.rs index 6a1f1879..b2e5a94c 100644 --- a/src/core/utils/hash/sha256.rs +++ b/src/core/utils/hash/sha256.rs @@ -1,6 +1,6 @@ use ring::{digest, digest::SHA256}; -#[tracing::instrument(skip_all)] +#[tracing::instrument(skip_all, level = "debug")] pub(super) fn hash(keys: &[&[u8]]) -> Vec { // We only hash the pdu's event ids, not the whole pdu let bytes = keys.join(&0xFF); diff --git a/src/router/request.rs b/src/router/request.rs index 2bdd06f8..49a31c29 100644 --- a/src/router/request.rs +++ b/src/router/request.rs @@ -9,7 +9,7 @@ use ruma::api::client::{ }; use tracing::{debug, error, trace}; -#[tracing::instrument(skip_all)] +#[tracing::instrument(skip_all, level = "debug")] pub(crate) async fn spawn( State(server): State>, req: http::Request, next: axum::middleware::Next, ) -> Result { diff --git a/src/service/account_data/mod.rs b/src/service/account_data/mod.rs index 66474370..69d2f799 100644 --- a/src/service/account_data/mod.rs +++ b/src/service/account_data/mod.rs @@ -44,7 +44,7 @@ impl Service { } /// Returns all changes to the account data that happened after `since`. - #[tracing::instrument(skip_all, name = "since")] + #[tracing::instrument(skip_all, name = "since", level = "debug")] pub fn changes_since( &self, room_id: Option<&RoomId>, user_id: &UserId, since: u64, ) -> Result>> { diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index 083aec3c..254a3d9c 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -110,7 +110,7 @@ impl Data { Ok(()) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub async fn watch(&self, user_id: &UserId, device_id: &DeviceId) -> Result<()> { let userid_bytes = user_id.as_bytes().to_vec(); let mut userid_prefix = userid_bytes.clone(); diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index f28299b2..11bfc88c 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -189,10 +189,10 @@ impl Service { #[inline] pub fn current_count(&self) -> Result { Ok(self.db.current_count()) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn last_check_for_updates_id(&self) -> Result { self.db.last_check_for_updates_id() } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn update_check_for_updates_id(&self, id: u64) -> Result<()> { self.db.update_check_for_updates_id(id) } pub async fn watch(&self, user_id: &UserId, device_id: &DeviceId) -> Result<()> { diff --git a/src/service/pusher/mod.rs b/src/service/pusher/mod.rs index 27b49c28..ea48ea7c 100644 --- a/src/service/pusher/mod.rs +++ b/src/service/pusher/mod.rs @@ -186,7 +186,7 @@ impl Service { Ok(()) } - #[tracing::instrument(skip(self, user, ruleset, pdu))] + #[tracing::instrument(skip(self, user, ruleset, pdu), level = "debug")] pub fn get_actions<'a>( &self, user: &UserId, ruleset: &'a Ruleset, power_levels: &RoomPowerLevelsEventContent, pdu: &Raw, room_id: &RoomId, diff --git a/src/service/rooms/alias/mod.rs b/src/service/rooms/alias/mod.rs index d375561e..706bf2f8 100644 --- a/src/service/rooms/alias/mod.rs +++ b/src/service/rooms/alias/mod.rs @@ -89,19 +89,19 @@ impl Service { ) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn resolve_local_alias(&self, alias: &RoomAliasId) -> Result> { self.db.resolve_local_alias(alias) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn local_aliases_for_room<'a>( &'a self, room_id: &RoomId, ) -> Box> + 'a> { self.db.local_aliases_for_room(room_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn all_local_aliases<'a>(&'a self) -> Box> + 'a> { self.db.all_local_aliases() } diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index 7b2a4f01..2919ea7e 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -174,13 +174,13 @@ impl Service { self.db.get_cached_eventid_authchain(key) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn cache_auth_chain(&self, key: Vec, auth_chain: &HashSet) -> Result<()> { self.db .cache_auth_chain(key, auth_chain.iter().copied().collect::>()) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn cache_auth_chain_vec(&self, key: Vec, auth_chain: &Vec) -> Result<()> { self.db .cache_auth_chain(key, auth_chain.iter().copied().collect::>()) diff --git a/src/service/rooms/directory/mod.rs b/src/service/rooms/directory/mod.rs index 3e60831c..23ec6b6b 100644 --- a/src/service/rooms/directory/mod.rs +++ b/src/service/rooms/directory/mod.rs @@ -22,15 +22,15 @@ impl crate::Service for Service { } impl Service { - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn set_public(&self, room_id: &RoomId) -> Result<()> { self.db.set_public(room_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn set_not_public(&self, room_id: &RoomId) -> Result<()> { self.db.set_not_public(room_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn is_public_room(&self, room_id: &RoomId) -> Result { self.db.is_public_room(room_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn public_rooms(&self) -> impl Iterator> + '_ { self.db.public_rooms() } } diff --git a/src/service/rooms/event_handler/signing_keys.rs b/src/service/rooms/event_handler/signing_keys.rs index fdb2be0e..2fa5b0df 100644 --- a/src/service/rooms/event_handler/signing_keys.rs +++ b/src/service/rooms/event_handler/signing_keys.rs @@ -3,6 +3,7 @@ use std::{ time::{Duration, SystemTime}, }; +use conduit::{debug, error, info, trace, warn}; use futures_util::{stream::FuturesUnordered, StreamExt}; use ruma::{ api::federation::{ @@ -19,7 +20,6 @@ use ruma::{ }; use serde_json::value::RawValue as RawJsonValue; use tokio::sync::{RwLock, RwLockWriteGuard}; -use tracing::{debug, error, info, trace, warn}; use crate::{services, Error, Result}; diff --git a/src/service/rooms/lazy_loading/mod.rs b/src/service/rooms/lazy_loading/mod.rs index 185cfd8c..96f623f2 100644 --- a/src/service/rooms/lazy_loading/mod.rs +++ b/src/service/rooms/lazy_loading/mod.rs @@ -39,7 +39,7 @@ impl crate::Service for Service { } impl Service { - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn lazy_load_was_sent_before( &self, user_id: &UserId, device_id: &DeviceId, room_id: &RoomId, ll_user: &UserId, ) -> Result { @@ -47,7 +47,7 @@ impl Service { .lazy_load_was_sent_before(user_id, device_id, room_id, ll_user) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub async fn lazy_load_mark_sent( &self, user_id: &UserId, device_id: &DeviceId, room_id: &RoomId, lazy_load: HashSet, count: PduCount, @@ -58,7 +58,7 @@ impl Service { .insert((user_id.to_owned(), device_id.to_owned(), room_id.to_owned(), count), lazy_load); } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub async fn lazy_load_confirm_delivery( &self, user_id: &UserId, device_id: &DeviceId, room_id: &RoomId, since: PduCount, ) -> Result<()> { @@ -77,7 +77,7 @@ impl Service { Ok(()) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn lazy_load_reset(&self, user_id: &UserId, device_id: &DeviceId, room_id: &RoomId) -> Result<()> { self.db.lazy_load_reset(user_id, device_id, room_id) } diff --git a/src/service/rooms/outlier/mod.rs b/src/service/rooms/outlier/mod.rs index a7326f42..22bd2092 100644 --- a/src/service/rooms/outlier/mod.rs +++ b/src/service/rooms/outlier/mod.rs @@ -35,7 +35,7 @@ impl Service { pub fn get_pdu_outlier(&self, event_id: &EventId) -> Result> { self.db.get_outlier_pdu(event_id) } /// Append the PDU as an outlier. - #[tracing::instrument(skip(self, pdu))] + #[tracing::instrument(skip(self, pdu), level = "debug")] pub fn add_pdu_outlier(&self, event_id: &EventId, pdu: &CanonicalJsonObject) -> Result<()> { self.db.add_pdu_outlier(event_id, pdu) } diff --git a/src/service/rooms/pdu_metadata/mod.rs b/src/service/rooms/pdu_metadata/mod.rs index cdb2fc29..05067aa8 100644 --- a/src/service/rooms/pdu_metadata/mod.rs +++ b/src/service/rooms/pdu_metadata/mod.rs @@ -38,7 +38,7 @@ impl crate::Service for Service { } impl Service { - #[tracing::instrument(skip(self, from, to))] + #[tracing::instrument(skip(self, from, to), level = "debug")] pub fn add_relation(&self, from: PduCount, to: PduCount) -> Result<()> { match (from, to) { (PduCount::Normal(f), PduCount::Normal(t)) => self.db.add_relation(f, t), @@ -218,19 +218,19 @@ impl Service { }) } - #[tracing::instrument(skip(self, room_id, event_ids))] + #[tracing::instrument(skip_all, level = "debug")] pub fn mark_as_referenced(&self, room_id: &RoomId, event_ids: &[Arc]) -> Result<()> { self.db.mark_as_referenced(room_id, event_ids) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn is_event_referenced(&self, room_id: &RoomId, event_id: &EventId) -> Result { self.db.is_event_referenced(room_id, event_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn mark_event_soft_failed(&self, event_id: &EventId) -> Result<()> { self.db.mark_event_soft_failed(event_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn is_event_soft_failed(&self, event_id: &EventId) -> Result { self.db.is_event_soft_failed(event_id) } } diff --git a/src/service/rooms/read_receipt/mod.rs b/src/service/rooms/read_receipt/mod.rs index e46027b7..9375276e 100644 --- a/src/service/rooms/read_receipt/mod.rs +++ b/src/service/rooms/read_receipt/mod.rs @@ -33,7 +33,7 @@ impl Service { /// Returns an iterator over the most recent read_receipts in a room that /// happened after the event with id `since`. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn readreceipts_since<'a>( &'a self, room_id: &RoomId, since: u64, ) -> impl Iterator)>> + 'a { @@ -41,13 +41,13 @@ impl Service { } /// Sets a private read marker at `count`. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn private_read_set(&self, room_id: &RoomId, user_id: &UserId, count: u64) -> Result<()> { self.db.private_read_set(room_id, user_id, count) } /// Returns the private read marker. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn private_read_get(&self, room_id: &RoomId, user_id: &UserId) -> Result> { self.db.private_read_get(room_id, user_id) } diff --git a/src/service/rooms/search/mod.rs b/src/service/rooms/search/mod.rs index 7573d218..082dd432 100644 --- a/src/service/rooms/search/mod.rs +++ b/src/service/rooms/search/mod.rs @@ -21,17 +21,17 @@ impl crate::Service for Service { } impl Service { - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn index_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) -> Result<()> { self.db.index_pdu(shortroomid, pdu_id, message_body) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn deindex_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) -> Result<()> { self.db.deindex_pdu(shortroomid, pdu_id, message_body) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn search_pdus<'a>( &'a self, room_id: &RoomId, search_string: &str, ) -> Result> + 'a, Vec)>> { diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index 7964a5fb..52ee89d1 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -116,7 +116,7 @@ impl Service { /// /// This adds all current state events (not including the incoming event) /// to `stateid_pduid` and adds the incoming event to `eventid_statehash`. - #[tracing::instrument(skip(self, state_ids_compressed))] + #[tracing::instrument(skip(self, state_ids_compressed), level = "debug")] pub fn set_event_state( &self, event_id: &EventId, room_id: &RoomId, state_ids_compressed: Arc>, ) -> Result { @@ -184,7 +184,7 @@ impl Service { /// /// This adds all current state events (not including the incoming event) /// to `stateid_pduid` and adds the incoming event to `eventid_statehash`. - #[tracing::instrument(skip(self, new_pdu))] + #[tracing::instrument(skip(self, new_pdu), level = "debug")] pub fn append_to_state(&self, new_pdu: &PduEvent) -> Result { let shorteventid = services() .rooms @@ -257,7 +257,7 @@ impl Service { } } - #[tracing::instrument(skip(self, invite_event))] + #[tracing::instrument(skip(self, invite_event), level = "debug")] pub fn calculate_invite_state(&self, invite_event: &PduEvent) -> Result>> { let mut state = Vec::new(); // Add recommended events @@ -313,7 +313,7 @@ impl Service { } /// Set the state hash to a new version, but does not update state_cache. - #[tracing::instrument(skip(self, mutex_lock))] + #[tracing::instrument(skip(self, mutex_lock), level = "debug")] pub fn set_room_state( &self, room_id: &RoomId, @@ -324,7 +324,7 @@ impl Service { } /// Returns the room's version. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn get_room_version(&self, room_id: &RoomId) -> Result { let create_event = services() .rooms @@ -365,7 +365,7 @@ impl Service { } /// This fetches auth events from the current state. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn get_auth_events( &self, room_id: &RoomId, kind: &TimelineEventType, sender: &UserId, state_key: Option<&str>, content: &serde_json::value::RawValue, diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index e8554500..a3567857 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -81,7 +81,7 @@ impl crate::Service for Service { impl Service { /// Builds a StateMap by iterating over all keys that start /// with state_hash, this gives the full state for the given state_hash. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub async fn state_full_ids(&self, shortstatehash: u64) -> Result>> { self.db.state_full_ids(shortstatehash).await } @@ -92,7 +92,7 @@ impl Service { /// Returns a single PDU from `room_id` with key (`event_type`, /// `state_key`). - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn state_get_id( &self, shortstatehash: u64, event_type: &StateEventType, state_key: &str, ) -> Result>> { @@ -286,14 +286,14 @@ impl Service { pub fn pdu_shortstatehash(&self, event_id: &EventId) -> Result> { self.db.pdu_shortstatehash(event_id) } /// Returns the full room state. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub async fn room_state_full(&self, room_id: &RoomId) -> Result>> { self.db.room_state_full(room_id).await } /// Returns a single PDU from `room_id` with key (`event_type`, /// `state_key`). - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn room_state_get_id( &self, room_id: &RoomId, event_type: &StateEventType, state_key: &str, ) -> Result>> { @@ -302,7 +302,7 @@ impl Service { /// Returns a single PDU from `room_id` with key (`event_type`, /// `state_key`). - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn room_state_get( &self, room_id: &RoomId, event_type: &StateEventType, state_key: &str, ) -> Result>> { diff --git a/src/service/rooms/state_cache/data.rs b/src/service/rooms/state_cache/data.rs index 0120d5a6..2b9fbe94 100644 --- a/src/service/rooms/state_cache/data.rs +++ b/src/service/rooms/state_cache/data.rs @@ -213,7 +213,7 @@ impl Data { Ok(()) } - #[tracing::instrument(skip(self, room_id, appservice))] + #[tracing::instrument(skip(self, room_id, appservice), level = "debug")] pub(super) fn appservice_in_room(&self, room_id: &RoomId, appservice: &RegistrationInfo) -> Result { let maybe = self .appservice_in_room_cache @@ -249,7 +249,7 @@ impl Data { } /// Makes a user forget a room. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn forget(&self, room_id: &RoomId, user_id: &UserId) -> Result<()> { let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xFF); @@ -266,7 +266,7 @@ impl Data { } /// Returns an iterator of all servers participating in this room. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn room_servers<'a>( &'a self, room_id: &RoomId, ) -> Box> + 'a> { @@ -286,7 +286,7 @@ impl Data { })) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn server_in_room(&self, server: &ServerName, room_id: &RoomId) -> Result { let mut key = server.as_bytes().to_vec(); key.push(0xFF); @@ -297,7 +297,7 @@ impl Data { /// Returns an iterator of all rooms a server participates in (as far as we /// know). - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn server_rooms<'a>( &'a self, server: &ServerName, ) -> Box> + 'a> { @@ -318,7 +318,7 @@ impl Data { } /// Returns an iterator of all joined members of a room. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn room_members<'a>( &'a self, room_id: &RoomId, ) -> Box> + Send + 'a> { @@ -350,7 +350,7 @@ impl Data { /// Returns an iterator of all our local joined users in a room who are /// active (not deactivated, not guest) - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn active_local_users_in_room<'a>( &'a self, room_id: &RoomId, ) -> Box + 'a> { @@ -361,7 +361,7 @@ impl Data { } /// Returns the number of users which are currently in a room - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn room_joined_count(&self, room_id: &RoomId) -> Result> { self.roomid_joinedcount .get(room_id.as_bytes())? @@ -370,7 +370,7 @@ impl Data { } /// Returns the number of users which are currently invited to a room - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn room_invited_count(&self, room_id: &RoomId) -> Result> { self.roomid_invitedcount .get(room_id.as_bytes())? @@ -379,7 +379,7 @@ impl Data { } /// Returns an iterator over all User IDs who ever joined a room. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn room_useroncejoined<'a>( &'a self, room_id: &RoomId, ) -> Box> + 'a> { @@ -404,7 +404,7 @@ impl Data { } /// Returns an iterator over all invited members of a room. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn room_members_invited<'a>( &'a self, room_id: &RoomId, ) -> Box> + 'a> { @@ -428,7 +428,7 @@ impl Data { ) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn get_invite_count(&self, room_id: &RoomId, user_id: &UserId) -> Result> { let mut key = room_id.as_bytes().to_vec(); key.push(0xFF); @@ -443,7 +443,7 @@ impl Data { }) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn get_left_count(&self, room_id: &RoomId, user_id: &UserId) -> Result> { let mut key = room_id.as_bytes().to_vec(); key.push(0xFF); @@ -456,7 +456,7 @@ impl Data { } /// Returns an iterator over all rooms this user joined. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn rooms_joined(&self, user_id: &UserId) -> Box> + '_> { Box::new( self.userroomid_joined @@ -476,7 +476,7 @@ impl Data { } /// Returns an iterator over all rooms a user was invited to. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn rooms_invited<'a>(&'a self, user_id: &UserId) -> StrippedStateEventIter<'a> { let mut prefix = user_id.as_bytes().to_vec(); prefix.push(0xFF); @@ -503,7 +503,7 @@ impl Data { ) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn invite_state( &self, user_id: &UserId, room_id: &RoomId, ) -> Result>>> { @@ -522,7 +522,7 @@ impl Data { .transpose() } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn left_state( &self, user_id: &UserId, room_id: &RoomId, ) -> Result>>> { @@ -542,7 +542,7 @@ impl Data { } /// Returns an iterator over all rooms a user left. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn rooms_left<'a>(&'a self, user_id: &UserId) -> AnySyncStateEventIter<'a> { let mut prefix = user_id.as_bytes().to_vec(); prefix.push(0xFF); @@ -569,7 +569,7 @@ impl Data { ) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn once_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result { let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xFF); @@ -578,7 +578,7 @@ impl Data { Ok(self.roomuseroncejoinedids.get(&userroom_id)?.is_some()) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn is_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result { let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xFF); @@ -587,7 +587,7 @@ impl Data { Ok(self.userroomid_joined.get(&userroom_id)?.is_some()) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn is_invited(&self, user_id: &UserId, room_id: &RoomId) -> Result { let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xFF); @@ -596,7 +596,7 @@ impl Data { Ok(self.userroomid_invitestate.get(&userroom_id)?.is_some()) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn is_left(&self, user_id: &UserId, room_id: &RoomId) -> Result { let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xFF); @@ -605,7 +605,7 @@ impl Data { Ok(self.userroomid_leftstate.get(&userroom_id)?.is_some()) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn servers_invite_via<'a>( &'a self, room_id: &RoomId, ) -> Box> + 'a> { @@ -631,7 +631,7 @@ impl Data { ) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn add_servers_invite_via(&self, room_id: &RoomId, servers: &[OwnedServerName]) -> Result<()> { let mut prev_servers = self .servers_invite_via(room_id) diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index a0910c7c..30fd7bf4 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -219,10 +219,10 @@ impl Service { Ok(()) } - #[tracing::instrument(skip(self, room_id))] + #[tracing::instrument(skip(self, room_id), level = "debug")] pub fn update_joined_count(&self, room_id: &RoomId) -> Result<()> { self.db.update_joined_count(room_id) } - #[tracing::instrument(skip(self, room_id, appservice))] + #[tracing::instrument(skip(self, room_id, appservice), level = "debug")] pub fn appservice_in_room(&self, room_id: &RoomId, appservice: &RegistrationInfo) -> Result { self.db.appservice_in_room(room_id, appservice) } @@ -230,7 +230,7 @@ impl Service { /// Direct DB function to directly mark a user as left. It is not /// recommended to use this directly. You most likely should use /// `update_membership` instead - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn mark_as_left(&self, user_id: &UserId, room_id: &RoomId) -> Result<()> { self.db.mark_as_left(user_id, room_id) } @@ -238,35 +238,35 @@ impl Service { /// Direct DB function to directly mark a user as joined. It is not /// recommended to use this directly. You most likely should use /// `update_membership` instead - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn mark_as_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result<()> { self.db.mark_as_joined(user_id, room_id) } /// Makes a user forget a room. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn forget(&self, room_id: &RoomId, user_id: &UserId) -> Result<()> { self.db.forget(room_id, user_id) } /// Returns an iterator of all servers participating in this room. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn room_servers(&self, room_id: &RoomId) -> impl Iterator> + '_ { self.db.room_servers(room_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn server_in_room(&self, server: &ServerName, room_id: &RoomId) -> Result { self.db.server_in_room(server, room_id) } /// Returns an iterator of all rooms a server participates in (as far as we /// know). - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn server_rooms(&self, server: &ServerName) -> impl Iterator> + '_ { self.db.server_rooms(server) } /// Returns true if server can see user by sharing at least one room. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn server_sees_user(&self, server: &ServerName, user_id: &UserId) -> Result { Ok(self .server_rooms(server) @@ -275,7 +275,7 @@ impl Service { } /// Returns true if user_a and user_b share at least one room. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn user_sees_user(&self, user_a: &UserId, user_b: &UserId) -> Result { // Minimize number of point-queries by iterating user with least nr rooms let (a, b) = if self.rooms_joined(user_a).count() < self.rooms_joined(user_b).count() { @@ -291,23 +291,23 @@ impl Service { } /// Returns an iterator over all joined members of a room. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn room_members(&self, room_id: &RoomId) -> impl Iterator> + Send + '_ { self.db.room_members(room_id) } /// Returns the number of users which are currently in a room - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn room_joined_count(&self, room_id: &RoomId) -> Result> { self.db.room_joined_count(room_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] /// Returns an iterator of all our local users in the room, even if they're /// deactivated/guests pub fn local_users_in_room<'a>(&'a self, room_id: &RoomId) -> impl Iterator + 'a { self.db.local_users_in_room(room_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] /// Returns an iterator of all our local joined users in a room who are /// active (not deactivated, not guest) pub fn active_local_users_in_room<'a>(&'a self, room_id: &RoomId) -> impl Iterator + 'a { @@ -315,80 +315,80 @@ impl Service { } /// Returns the number of users which are currently invited to a room - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn room_invited_count(&self, room_id: &RoomId) -> Result> { self.db.room_invited_count(room_id) } /// Returns an iterator over all User IDs who ever joined a room. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn room_useroncejoined(&self, room_id: &RoomId) -> impl Iterator> + '_ { self.db.room_useroncejoined(room_id) } /// Returns an iterator over all invited members of a room. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn room_members_invited(&self, room_id: &RoomId) -> impl Iterator> + '_ { self.db.room_members_invited(room_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn get_invite_count(&self, room_id: &RoomId, user_id: &UserId) -> Result> { self.db.get_invite_count(room_id, user_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn get_left_count(&self, room_id: &RoomId, user_id: &UserId) -> Result> { self.db.get_left_count(room_id, user_id) } /// Returns an iterator over all rooms this user joined. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn rooms_joined(&self, user_id: &UserId) -> impl Iterator> + '_ { self.db.rooms_joined(user_id) } /// Returns an iterator over all rooms a user was invited to. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn rooms_invited( &self, user_id: &UserId, ) -> impl Iterator>)>> + '_ { self.db.rooms_invited(user_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn invite_state(&self, user_id: &UserId, room_id: &RoomId) -> Result>>> { self.db.invite_state(user_id, room_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn left_state(&self, user_id: &UserId, room_id: &RoomId) -> Result>>> { self.db.left_state(user_id, room_id) } /// Returns an iterator over all rooms a user left. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn rooms_left( &self, user_id: &UserId, ) -> impl Iterator>)>> + '_ { self.db.rooms_left(user_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn once_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result { self.db.once_joined(user_id, room_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn is_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result { self.db.is_joined(user_id, room_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn is_invited(&self, user_id: &UserId, room_id: &RoomId) -> Result { self.db.is_invited(user_id, room_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn is_left(&self, user_id: &UserId, room_id: &RoomId) -> Result { self.db.is_left(user_id, room_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn servers_invite_via(&self, room_id: &RoomId) -> impl Iterator> + '_ { self.db.servers_invite_via(room_id) } diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index 197efbea..4b4ea7d4 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -77,7 +77,7 @@ impl crate::Service for Service { impl Service { /// Returns a stack with info on shortstatehash, full state, added diff and /// removed diff for the selected shortstatehash and each parent layer. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn load_shortstatehash_info(&self, shortstatehash: u64) -> ShortStateInfoResult { if let Some(r) = self .stateinfo_cache @@ -162,7 +162,7 @@ impl Service { /// for this layer /// * `parent_states` - A stack with info on shortstatehash, full state, /// added diff and removed diff for each parent layer - #[tracing::instrument(skip(self, statediffnew, statediffremoved, diff_to_sibling, parent_states))] + #[tracing::instrument(skip(self, statediffnew, statediffremoved, diff_to_sibling, parent_states), level = "debug")] pub fn save_state_from_diff( &self, shortstatehash: u64, statediffnew: Arc>, statediffremoved: Arc>, diff_to_sibling: usize, diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 70f4423c..db7af004 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -99,7 +99,7 @@ impl crate::Service for Service { } impl Service { - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn first_pdu_in_room(&self, room_id: &RoomId) -> Result>> { self.all_pdus(user_id!("@doesntmatter:conduit.rs"), room_id)? .next() @@ -107,7 +107,7 @@ impl Service { .transpose() } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn latest_pdu_in_room(&self, room_id: &RoomId) -> Result>> { self.all_pdus(user_id!("@placeholder:conduwuit.placeholder"), room_id)? .last() @@ -115,7 +115,7 @@ impl Service { .transpose() } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn last_timeline_count(&self, sender_user: &UserId, room_id: &RoomId) -> Result { self.db.last_timeline_count(sender_user, room_id) } @@ -213,7 +213,7 @@ impl Service { } /// Removes a pdu and creates a new one with the same id. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn replace_pdu(&self, pdu_id: &[u8], pdu_json: &CanonicalJsonObject, pdu: &PduEvent) -> Result<()> { self.db.replace_pdu(pdu_id, pdu_json, pdu) } @@ -1030,7 +1030,7 @@ impl Service { /// Returns an iterator over all events and their tokens in a room that /// happened before the event with id `until` in reverse-chronological /// order. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn pdus_until<'a>( &'a self, user_id: &UserId, room_id: &RoomId, until: PduCount, ) -> Result> + 'a> { @@ -1039,7 +1039,7 @@ impl Service { /// Returns an iterator over all events and their token in a room that /// happened after the event with id `from` in chronological order. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn pdus_after<'a>( &'a self, user_id: &UserId, room_id: &RoomId, from: PduCount, ) -> Result> + 'a> { diff --git a/src/service/sending/data.rs b/src/service/sending/data.rs index ed579817..65725618 100644 --- a/src/service/sending/data.rs +++ b/src/service/sending/data.rs @@ -136,7 +136,7 @@ impl Data { } } -#[tracing::instrument(skip(key))] +#[tracing::instrument(skip(key), level = "debug")] fn parse_servercurrentevent(key: &[u8], value: Vec) -> Result<(Destination, SendingEvent)> { // Appservices start with a plus Ok::<_, Error>(if key.starts_with(b"+") { diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index 0dcebcec..a18240bd 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -93,7 +93,7 @@ impl crate::Service for Service { } impl Service { - #[tracing::instrument(skip(self, pdu_id, user, pushkey))] + #[tracing::instrument(skip(self, pdu_id, user, pushkey), level = "debug")] pub fn send_pdu_push(&self, pdu_id: &[u8], user: &UserId, pushkey: String) -> Result<()> { let dest = Destination::Push(user.to_owned(), pushkey); let event = SendingEvent::Pdu(pdu_id.to_owned()); @@ -106,7 +106,7 @@ impl Service { }) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn send_pdu_appservice(&self, appservice_id: String, pdu_id: Vec) -> Result<()> { let dest = Destination::Appservice(appservice_id); let event = SendingEvent::Pdu(pdu_id); @@ -119,7 +119,7 @@ impl Service { }) } - #[tracing::instrument(skip(self, room_id, pdu_id))] + #[tracing::instrument(skip(self, room_id, pdu_id), level = "debug")] pub fn send_pdu_room(&self, room_id: &RoomId, pdu_id: &[u8]) -> Result<()> { let servers = services() .rooms @@ -131,7 +131,7 @@ impl Service { self.send_pdu_servers(servers, pdu_id) } - #[tracing::instrument(skip(self, servers, pdu_id))] + #[tracing::instrument(skip(self, servers, pdu_id), level = "debug")] pub fn send_pdu_servers>(&self, servers: I, pdu_id: &[u8]) -> Result<()> { let requests = servers .into_iter() @@ -155,7 +155,7 @@ impl Service { Ok(()) } - #[tracing::instrument(skip(self, server, serialized))] + #[tracing::instrument(skip(self, server, serialized), level = "debug")] pub fn send_edu_server(&self, server: &ServerName, serialized: Vec) -> Result<()> { let dest = Destination::Normal(server.to_owned()); let event = SendingEvent::Edu(serialized); @@ -168,7 +168,7 @@ impl Service { }) } - #[tracing::instrument(skip(self, room_id, serialized))] + #[tracing::instrument(skip(self, room_id, serialized), level = "debug")] pub fn send_edu_room(&self, room_id: &RoomId, serialized: Vec) -> Result<()> { let servers = services() .rooms @@ -180,7 +180,7 @@ impl Service { self.send_edu_servers(servers, serialized) } - #[tracing::instrument(skip(self, servers, serialized))] + #[tracing::instrument(skip(self, servers, serialized), level = "debug")] pub fn send_edu_servers>(&self, servers: I, serialized: Vec) -> Result<()> { let requests = servers .into_iter() @@ -205,7 +205,7 @@ impl Service { Ok(()) } - #[tracing::instrument(skip(self, room_id))] + #[tracing::instrument(skip(self, room_id), level = "debug")] pub fn flush_room(&self, room_id: &RoomId) -> Result<()> { let servers = services() .rooms @@ -217,7 +217,7 @@ impl Service { self.flush_servers(servers) } - #[tracing::instrument(skip(self, servers))] + #[tracing::instrument(skip(self, servers), level = "debug")] pub fn flush_servers>(&self, servers: I) -> Result<()> { let requests = servers.into_iter().map(Destination::Normal); for dest in requests { @@ -255,7 +255,7 @@ impl Service { /// Cleanup event data /// Used for instance after we remove an appservice registration - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn cleanup_events(&self, appservice_id: String) -> Result<()> { self.db .delete_all_requests_for(&Destination::Appservice(appservice_id))?; @@ -271,7 +271,7 @@ impl Service { } impl Destination { - #[tracing::instrument(skip(self))] + #[must_use] pub fn get_prefix(&self) -> Vec { let mut prefix = match self { Self::Appservice(server) => { From 8e3be6feb0ff01d83122c26fcf071fa2922ff2bd Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 7 Jul 2024 19:43:41 +0000 Subject: [PATCH 040/158] slightly optimize Destination::get_prefix() Signed-off-by: Jason Volk --- src/service/sending/mod.rs | 44 +++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index a18240bd..d7a9c0fc 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -273,27 +273,45 @@ impl Service { impl Destination { #[must_use] pub fn get_prefix(&self) -> Vec { - let mut prefix = match self { - Self::Appservice(server) => { - let mut p = b"+".to_vec(); + match self { + Self::Normal(server) => { + let len = server.as_bytes().len().saturating_add(1); + + let mut p = Vec::with_capacity(len); p.extend_from_slice(server.as_bytes()); + p.push(0xFF); + p + }, + Self::Appservice(server) => { + let sigil = b"+"; + let len = sigil + .len() + .saturating_add(server.as_bytes().len()) + .saturating_add(1); + + let mut p = Vec::with_capacity(len); + p.extend_from_slice(sigil); + p.extend_from_slice(server.as_bytes()); + p.push(0xFF); p }, Self::Push(user, pushkey) => { - let mut p = b"$".to_vec(); + let sigil = b"$"; + let len = sigil + .len() + .saturating_add(user.as_bytes().len()) + .saturating_add(1) + .saturating_add(pushkey.as_bytes().len()) + .saturating_add(1); + + let mut p = Vec::with_capacity(len); + p.extend_from_slice(sigil); p.extend_from_slice(user.as_bytes()); p.push(0xFF); p.extend_from_slice(pushkey.as_bytes()); + p.push(0xFF); p }, - Self::Normal(server) => { - let mut p = Vec::new(); - p.extend_from_slice(server.as_bytes()); - p - }, - }; - prefix.push(0xFF); - - prefix + } } } From 8296e0ed676f9a4f152de9d2f0e4ae0c1688341d Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 8 Jul 2024 02:59:58 +0000 Subject: [PATCH 041/158] slightly optimize command completer Signed-off-by: Jason Volk --- src/admin/handler.rs | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/admin/handler.rs b/src/admin/handler.rs index 1466c040..22ec81c5 100644 --- a/src/admin/handler.rs +++ b/src/admin/handler.rs @@ -132,11 +132,13 @@ fn parse_admin_command(command_line: &str) -> Result { } fn complete_admin_command(mut cmd: clap::Command, line: &str) -> String { - let mut ret = Vec::::new(); let argv = parse_command_line(line); + let mut ret = Vec::::with_capacity(argv.len().saturating_add(1)); + 'token: for token in argv.into_iter().skip(1) { - let mut choice = Vec::new(); let cmd_ = cmd.clone(); + let mut choice = Vec::new(); + for sub in cmd_.get_subcommands() { let name = sub.get_name(); if *name == token { @@ -144,20 +146,20 @@ fn complete_admin_command(mut cmd: clap::Command, line: &str) -> String { ret.push(token); cmd.clone_from(sub); continue 'token; - } - if name.starts_with(&token) { + } else if name.starts_with(&token) { // partial match; add to choices choice.push(name); } } - if choice.is_empty() { + if choice.len() == 1 { + // One choice. Add extra space because it's complete + let choice = *choice.first().expect("only choice"); + ret.push(choice.to_owned()); + ret.push(String::new()); + } else if choice.is_empty() { // Nothing found, return original string ret.push(token); - } else if choice.len() == 1 { - // One choice. Add extra space because it's complete - ret.push((*choice.first().expect("only choice")).to_owned()); - ret.push(String::new()); } else { // Find the common prefix ret.push(common_prefix(&choice).into()); @@ -168,9 +170,8 @@ fn complete_admin_command(mut cmd: clap::Command, line: &str) -> String { } // Return from no completion. Needs a space though. - let mut ret = ret.join(" "); - ret.push(' '); - ret + ret.push(String::new()); + ret.join(" ") } // Parse chat messages from the admin room into an AdminCommand object From eb5556e74ee3d98d1cd0fcf78b908d0e3c482b31 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 7 Jul 2024 20:58:26 +0000 Subject: [PATCH 042/158] additional tracing spans / log cleanup. Signed-off-by: Jason Volk --- src/database/engine.rs | 18 +++++++++++------- src/database/maps.rs | 1 + 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/database/engine.rs b/src/database/engine.rs index 53e39abc..d02ecf58 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -30,6 +30,7 @@ pub struct Engine { pub(crate) type Db = DBWithThreadMode; impl Engine { + #[tracing::instrument(skip_all)] pub(crate) fn open(server: &Arc) -> Result> { let config = &server.config; let cache_capacity_bytes = config.db_cache_capacity_mb * 1024.0 * 1024.0; @@ -51,7 +52,7 @@ impl Engine { if config.rocksdb_repair { warn!("Starting database repair. This may take a long time..."); if let Err(e) = Db::repair(&db_opts, &config.database_path) { - error!("Repair failed: {:?}", e); + error!("Repair failed: {e:?}"); } } @@ -76,9 +77,9 @@ impl Engine { let db = res.or_else(or_else)?; info!( - "Opened database at sequence number {} in {:?}", - db.latest_sequence_number(), - load_time.elapsed() + sequence = %db.latest_sequence_number(), + time = ?load_time.elapsed(), + "Opened database." ); Ok(Arc::new(Self { @@ -93,15 +94,16 @@ impl Engine { })) } + #[tracing::instrument(skip(self))] pub(crate) fn open_cf(&self, name: &str) -> Result>> { let mut cfs = self.cfs.lock().expect("locked"); if !cfs.contains(name) { - debug!("Creating new column family in database: {}", name); + debug!("Creating new column family in database: {name}"); let mut col_cache = self.col_cache.write().expect("locked"); let opts = cf_options(&self.server.config, name, self.opts.clone(), &mut col_cache); if let Err(e) = self.db.create_cf(name, &opts) { - error!("Failed to create new column family: {e}"); + error!(?name, "Failed to create new column family: {e}"); return or_else(e); } @@ -148,18 +150,20 @@ impl Engine { )?; for (name, cache) in &*self.col_cache.read().expect("locked") { - writeln!(res, "{} cache: {:.2} MiB", name, mibs(u64::try_from(cache.get_usage())?))?; + writeln!(res, "{name} cache: {:.2} MiB", mibs(u64::try_from(cache.get_usage())?))?; } Ok(res) } + #[tracing::instrument(skip(self), level = "debug")] pub fn cleanup(&self) -> Result<()> { debug!("Running flush_opt"); let flushoptions = rocksdb::FlushOptions::default(); result(DBCommon::flush_opt(&self.db, &flushoptions)) } + #[tracing::instrument(skip(self))] pub fn backup(&self) -> Result<(), Box> { let config = &self.server.config; let path = config.database_backup_path.as_ref(); diff --git a/src/database/maps.rs b/src/database/maps.rs index 1e09041c..de78eaed 100644 --- a/src/database/maps.rs +++ b/src/database/maps.rs @@ -8,6 +8,7 @@ pub type Maps = BTreeMap>; pub(crate) fn open(db: &Arc) -> Result { open_list(db, MAPS) } +#[tracing::instrument(skip_all, level = "debug")] pub(crate) fn open_list(db: &Arc, maps: &[&str]) -> Result { Ok(maps .iter() From 100c6f572bee8fac288290674cfa16eb62aec71a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 8 Jul 2024 01:28:38 +0000 Subject: [PATCH 043/158] trim unused dependencies in member crates Signed-off-by: Jason Volk --- Cargo.lock | 9 --------- src/admin/Cargo.toml | 5 ----- src/api/Cargo.toml | 1 - src/database/Cargo.toml | 1 - src/router/Cargo.toml | 2 -- 5 files changed, 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7152c66d..36da41e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -612,14 +612,9 @@ dependencies = [ "clap", "conduit_api", "conduit_core", - "conduit_database", "conduit_service", - "futures-util", "log", - "loole", - "regex", "ruma", - "serde", "serde_json", "serde_yaml", "tokio", @@ -654,7 +649,6 @@ dependencies = [ "serde_html_form", "serde_json", "sha-1", - "thiserror", "tokio", "tracing", "webpage", @@ -706,7 +700,6 @@ version = "0.4.5" dependencies = [ "conduit_core", "log", - "ruma", "rust-rocksdb-uwu", "tokio", "tracing", @@ -724,14 +717,12 @@ dependencies = [ "conduit_admin", "conduit_api", "conduit_core", - "conduit_database", "conduit_service", "http 1.1.0", "http-body-util", "hyper 1.4.0", "hyper-util", "log", - "regex", "ruma", "sd-notify", "sentry", diff --git a/src/admin/Cargo.toml b/src/admin/Cargo.toml index 97de55ff..f84fbb92 100644 --- a/src/admin/Cargo.toml +++ b/src/admin/Cargo.toml @@ -29,15 +29,10 @@ release_max_log_level = [ clap.workspace = true conduit-api.workspace = true conduit-core.workspace = true -conduit-database.workspace = true conduit-service.workspace = true -futures-util.workspace = true log.workspace = true -loole.workspace = true -regex.workspace = true ruma.workspace = true serde_json.workspace = true -serde.workspace = true serde_yaml.workspace = true tokio.workspace = true tracing-subscriber.workspace = true diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index a27924c6..2f8bce54 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -56,7 +56,6 @@ serde_html_form.workspace = true serde_json.workspace = true serde.workspace = true sha-1.workspace = true -thiserror.workspace = true tokio.workspace = true tracing.workspace = true webpage.workspace = true diff --git a/src/database/Cargo.toml b/src/database/Cargo.toml index 6e95236b..8b0d3fc3 100644 --- a/src/database/Cargo.toml +++ b/src/database/Cargo.toml @@ -37,7 +37,6 @@ zstd_compression = [ [dependencies] conduit-core.workspace = true log.workspace = true -ruma.workspace = true rust-rocksdb.workspace = true tokio.workspace = true tracing.workspace = true diff --git a/src/router/Cargo.toml b/src/router/Cargo.toml index ecb09dd7..5312984a 100644 --- a/src/router/Cargo.toml +++ b/src/router/Cargo.toml @@ -54,7 +54,6 @@ axum.workspace = true conduit-admin.workspace = true conduit-api.workspace = true conduit-core.workspace = true -conduit-database.workspace = true conduit-service.workspace = true log.workspace = true tokio.workspace = true @@ -65,7 +64,6 @@ http-body-util.workspace = true http.workspace = true hyper.workspace = true hyper-util.workspace = true -regex.workspace = true ruma.workspace = true sentry.optional = true sentry-tower.optional = true From 23a9055199a3e7fb876177ecad8c99d4e57b2513 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 8 Jul 2024 16:10:18 +0000 Subject: [PATCH 044/158] relax and de-clutter let_underscore_must_use Signed-off-by: Jason Volk --- Cargo.toml | 1 - src/admin/mod.rs | 2 -- src/api/client/sync.rs | 11 +++-------- src/router/request.rs | 1 - src/router/run.rs | 1 - src/router/serve/unix.rs | 1 - src/service/admin/console.rs | 4 ---- src/service/mod.rs | 1 - src/service/services.rs | 12 ++---------- 9 files changed, 5 insertions(+), 29 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index dad0adeb..1ef39370 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -777,7 +777,6 @@ fn_to_numeric_cast_any = "warn" format_push_string = "warn" get_unwrap = "warn" impl_trait_in_params = "warn" -let_underscore_must_use = "warn" let_underscore_untyped = "warn" lossy_float_literal = "warn" mem_forget = "warn" diff --git a/src/admin/mod.rs b/src/admin/mod.rs index f2e35d80..6a47bc74 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -28,7 +28,6 @@ mod_ctor! {} mod_dtor! {} /// Install the admin command handler -#[allow(clippy::let_underscore_must_use)] pub async fn init() { _ = services() .admin @@ -45,7 +44,6 @@ pub async fn init() { } /// Uninstall the admin command handler -#[allow(clippy::let_underscore_must_use)] pub async fn fini() { _ = services().admin.handle.write().await.take(); _ = services() diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index 9ef1fab0..59813cb6 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -307,10 +307,7 @@ pub(crate) async fn sync_events_route( duration = Duration::from_secs(30); } - #[allow(clippy::let_underscore_must_use)] - { - _ = tokio::time::timeout(duration, watcher).await; - } + _ = tokio::time::timeout(duration, watcher).await; } Ok(response) @@ -1567,10 +1564,8 @@ pub(crate) async fn sync_events_v4_route( if duration.as_secs() > 30 { duration = Duration::from_secs(30); } - #[allow(clippy::let_underscore_must_use)] - { - _ = tokio::time::timeout(duration, watcher).await; - } + + _ = tokio::time::timeout(duration, watcher).await; } Ok(sync_events::v4::Response { diff --git a/src/router/request.rs b/src/router/request.rs index 49a31c29..079106ef 100644 --- a/src/router/request.rs +++ b/src/router/request.rs @@ -71,7 +71,6 @@ fn handle_result( } } -#[allow(clippy::unnecessary_wraps)] fn handle_result_403( _method: &Method, _uri: &Uri, result: &axum::response::Response, ) -> Result { diff --git a/src/router/run.rs b/src/router/run.rs index 4f7853d8..3e09823a 100644 --- a/src/router/run.rs +++ b/src/router/run.rs @@ -17,7 +17,6 @@ use crate::{layers, serve}; /// Main loop base #[tracing::instrument(skip_all)] -#[allow(clippy::let_underscore_must_use)] // various of these are intended pub(crate) async fn run(server: Arc) -> Result<(), Error> { let app = layers::build(&server)?; diff --git a/src/router/serve/unix.rs b/src/router/serve/unix.rs index 6c406d28..8373b749 100644 --- a/src/router/serve/unix.rs +++ b/src/router/serve/unix.rs @@ -54,7 +54,6 @@ pub(super) async fn serve(server: &Arc, app: Router, mut shutdown: broad Ok(()) } -#[allow(clippy::let_underscore_must_use)] async fn accept( server: &Arc, listener: &UnixListener, tasks: &mut JoinSet<()>, mut app: MakeService, builder: server::conn::auto::Builder, conn: (UnixStream, SocketAddr), diff --git a/src/service/admin/console.rs b/src/service/admin/console.rs index 0a200cae..27a4b6c7 100644 --- a/src/service/admin/console.rs +++ b/src/service/admin/console.rs @@ -45,7 +45,6 @@ impl Console { } } - #[allow(clippy::let_underscore_must_use)] pub async fn start(self: &Arc) { let mut worker_join = self.worker_join.lock().expect("locked"); if worker_join.is_none() { @@ -54,7 +53,6 @@ impl Console { } } - #[allow(clippy::let_underscore_must_use)] pub async fn close(self: &Arc) { self.interrupt(); let Some(worker_join) = self.worker_join.lock().expect("locked").take() else { @@ -113,7 +111,6 @@ impl Console { self.worker_join.lock().expect("locked").take(); } - #[allow(clippy::let_underscore_must_use)] async fn readline(self: &Arc) -> Result { let _suppression = log::Suppress::new(&services().server); @@ -138,7 +135,6 @@ impl Console { result } - #[allow(clippy::let_underscore_must_use)] async fn handle(self: Arc, line: String) { if line.trim().is_empty() { return; diff --git a/src/service/mod.rs b/src/service/mod.rs index 9c83b25c..4b19073d 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -37,7 +37,6 @@ conduit::mod_dtor! {} static SERVICES: RwLock> = RwLock::new(None); -#[allow(clippy::let_underscore_must_use)] pub async fn init(server: &Arc) -> Result<()> { let d = Arc::new(Database::open(server).await?); let s = Box::new(Services::build(server.clone(), d)?); diff --git a/src/service/services.rs b/src/service/services.rs index 0ba86693..aeed8204 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -132,11 +132,7 @@ impl Services { if self.globals.allow_check_for_updates() { let handle = globals::updates::start_check_for_updates_task(); - - #[allow(clippy::let_underscore_must_use)] // needed for shutdown - { - _ = self.globals.updates_handle.lock().await.insert(handle); - } + _ = self.globals.updates_handle.lock().await.insert(handle); } debug_info!("Services startup complete."); @@ -159,11 +155,7 @@ impl Services { debug!("Waiting for update worker..."); if let Some(updates_handle) = self.globals.updates_handle.lock().await.take() { updates_handle.abort(); - - #[allow(clippy::let_underscore_must_use)] - { - _ = updates_handle.await; - } + _ = updates_handle.await; } for (name, service) in &self.service { From 51df946911204536fad2940d5d35904510ad3bb6 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 8 Jul 2024 16:13:31 +0000 Subject: [PATCH 045/158] de-branch duration limit statement Signed-off-by: Jason Volk --- src/api/client/sync.rs | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index 59813cb6..45a1c75b 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -1,4 +1,5 @@ use std::{ + cmp, cmp::Ordering, collections::{hash_map::Entry, BTreeMap, BTreeSet, HashMap, HashSet}, time::Duration, @@ -302,11 +303,8 @@ pub(crate) async fn sync_events_route( { // Hang a few seconds so requests are not spammed // Stop hanging if new info arrives - let mut duration = body.timeout.unwrap_or_default(); - if duration.as_secs() > 30 { - duration = Duration::from_secs(30); - } - + let default = Duration::from_secs(30); + let duration = cmp::min(body.timeout.unwrap_or(default), default); _ = tokio::time::timeout(duration, watcher).await; } @@ -1560,11 +1558,8 @@ pub(crate) async fn sync_events_v4_route( { // Hang a few seconds so requests are not spammed // Stop hanging if new info arrives - let mut duration = body.timeout.unwrap_or(Duration::from_secs(30)); - if duration.as_secs() > 30 { - duration = Duration::from_secs(30); - } - + let default = Duration::from_secs(30); + let duration = cmp::min(body.timeout.unwrap_or(default), default); _ = tokio::time::timeout(duration, watcher).await; } From 59d86d364137617e850a6ef6563aa954c8188587 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 8 Jul 2024 16:30:59 +0000 Subject: [PATCH 046/158] restrict clippy::string_slice Signed-off-by: Jason Volk --- Cargo.toml | 1 + src/core/config/proxy.rs | 1 + src/core/utils/html.rs | 1 + src/core/utils/string.rs | 5 +++-- src/service/sending/resolve.rs | 1 + 5 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 1ef39370..0d3d59fd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -790,6 +790,7 @@ rest_pat_in_fully_bound_structs = "warn" semicolon_outside_block = "warn" str_to_string = "warn" string_lit_chars_any = "warn" +string_slice = "warn" string_to_string = "warn" suspicious_xor_used_as_pow = "warn" tests_outside_test_module = "warn" diff --git a/src/core/config/proxy.rs b/src/core/config/proxy.rs index d823e5e4..48f883c6 100644 --- a/src/core/config/proxy.rs +++ b/src/core/config/proxy.rs @@ -127,6 +127,7 @@ impl WildCardedDomain { impl std::str::FromStr for WildCardedDomain { type Err = std::convert::Infallible; + #[allow(clippy::string_slice)] fn from_str(s: &str) -> Result { // maybe do some domain validation? Ok(if s.starts_with("*.") { diff --git a/src/core/utils/html.rs b/src/core/utils/html.rs index 938e50ec..fe07b2dd 100644 --- a/src/core/utils/html.rs +++ b/src/core/utils/html.rs @@ -6,6 +6,7 @@ pub struct Escape<'a>(pub &'a str); /// Copied from librustdoc: /// * +#[allow(clippy::string_slice)] impl fmt::Display for Escape<'_> { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { // Because the internet is always right, turns out there's not that many diff --git a/src/core/utils/string.rs b/src/core/utils/string.rs index 1f2a6572..ec423d53 100644 --- a/src/core/utils/string.rs +++ b/src/core/utils/string.rs @@ -9,12 +9,13 @@ pub const EMPTY: &str = ""; /// common_prefix(&input) == "con"; /// ``` #[must_use] +#[allow(clippy::string_slice)] pub fn common_prefix<'a>(choice: &'a [&str]) -> &'a str { choice.first().map_or(EMPTY, move |best| { choice.iter().skip(1).fold(*best, |best, choice| { &best[0..choice - .chars() - .zip(best.chars()) + .char_indices() + .zip(best.char_indices()) .take_while(|&(a, b)| a == b) .count()] }) diff --git a/src/service/sending/resolve.rs b/src/service/sending/resolve.rs index 77311006..d38509ba 100644 --- a/src/service/sending/resolve.rs +++ b/src/service/sending/resolve.rs @@ -484,6 +484,7 @@ impl FedDest { } #[inline] + #[allow(clippy::string_slice)] fn port(&self) -> Option { match &self { Self::Literal(addr) => Some(addr.port()), From 7ba0777bd304e09d8f34ac0e2e5714409d0772c7 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 8 Jul 2024 02:50:10 +0000 Subject: [PATCH 047/158] move RumaResponse out of core Error; cleanup Error conversions. Signed-off-by: Jason Volk --- Cargo.lock | 1 + src/api/Cargo.toml | 1 + src/api/client/unstable.rs | 4 +- src/api/mod.rs | 3 +- src/api/router/mod.rs | 3 +- src/api/router/response.rs | 22 +++++ src/core/error.rs | 192 +++++++++++++++++-------------------- src/core/mod.rs | 2 +- src/router/request.rs | 4 +- 9 files changed, 122 insertions(+), 110 deletions(-) create mode 100644 src/api/router/response.rs diff --git a/Cargo.lock b/Cargo.lock index 36da41e0..dd0c0436 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -637,6 +637,7 @@ dependencies = [ "futures-util", "hmac", "http 1.1.0", + "http-body-util", "hyper 1.4.0", "image", "ipaddress", diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index 2f8bce54..45cae73d 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -44,6 +44,7 @@ conduit-service.workspace = true futures-util.workspace = true hmac.workspace = true http.workspace = true +http-body-util.workspace = true hyper.workspace = true image.workspace = true ipaddress.workspace = true diff --git a/src/api/client/unstable.rs b/src/api/client/unstable.rs index 77cac0fa..e39db94e 100644 --- a/src/api/client/unstable.rs +++ b/src/api/client/unstable.rs @@ -1,12 +1,12 @@ use axum_client_ip::InsecureClientIp; -use conduit::{warn, RumaResponse}; +use conduit::warn; use ruma::{ api::client::{error::ErrorKind, membership::mutual_rooms, room::get_summary}, events::room::member::MembershipState, OwnedRoomId, }; -use crate::{services, Error, Result, Ruma}; +use crate::{services, Error, Result, Ruma, RumaResponse}; /// # `GET /_matrix/client/unstable/uk.half-shot.msc2666/user/mutual_rooms` /// diff --git a/src/api/mod.rs b/src/api/mod.rs index 8e30a518..0395da7a 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -9,7 +9,8 @@ extern crate conduit_service as service; pub(crate) use conduit::{debug_info, debug_warn, utils, Error, Result}; pub(crate) use service::{pdu::PduEvent, services, user_is_local}; -pub(crate) use crate::router::{Ruma, RumaResponse}; +pub(crate) use self::router::Ruma; +pub use self::router::RumaResponse; conduit::mod_ctor! {} conduit::mod_dtor! {} diff --git a/src/api/router/mod.rs b/src/api/router/mod.rs index 2c439d65..f167a606 100644 --- a/src/api/router/mod.rs +++ b/src/api/router/mod.rs @@ -1,12 +1,12 @@ mod auth; mod handler; mod request; +mod response; use std::{mem, ops::Deref}; use axum::{async_trait, body::Body, extract::FromRequest}; use bytes::{BufMut, BytesMut}; -pub(super) use conduit::error::RumaResponse; use conduit::{debug, debug_warn, trace, warn}; use ruma::{ api::{client::error::ErrorKind, IncomingRequest}, @@ -14,6 +14,7 @@ use ruma::{ }; pub(super) use self::handler::RouterExt; +pub use self::response::RumaResponse; use self::{auth::Auth, request::Request}; use crate::{service::appservice::RegistrationInfo, services, Error, Result}; diff --git a/src/api/router/response.rs b/src/api/router/response.rs new file mode 100644 index 00000000..38e58ba9 --- /dev/null +++ b/src/api/router/response.rs @@ -0,0 +1,22 @@ +use axum::response::{IntoResponse, Response}; +use bytes::BytesMut; +use conduit::Error; +use http::StatusCode; +use http_body_util::Full; +use ruma::api::{client::uiaa::UiaaResponse, OutgoingResponse}; + +#[derive(Clone)] +pub struct RumaResponse(pub T); + +impl From for RumaResponse { + fn from(t: Error) -> Self { Self(t.into()) } +} + +impl IntoResponse for RumaResponse { + fn into_response(self) -> Response { + match self.0.try_into_http_response::() { + Ok(res) => res.map(BytesMut::freeze).map(Full::new).into_response(), + Err(_) => StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + } +} diff --git a/src/core/error.rs b/src/core/error.rs index 63729f31..cb74d531 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -1,19 +1,11 @@ use std::{convert::Infallible, fmt}; -use axum::response::{IntoResponse, Response}; use bytes::BytesMut; use http::StatusCode; use http_body_util::Full; use ruma::{ api::{ - client::{ - error::ErrorKind::{ - Forbidden, GuestAccessForbidden, LimitExceeded, MissingToken, NotFound, ThreepidAuthFailed, - ThreepidDenied, TooLarge, Unauthorized, Unknown, UnknownToken, Unrecognized, UserDeactivated, - WrongRoomKeysVersion, - }, - uiaa::{UiaaInfo, UiaaResponse}, - }, + client::uiaa::{UiaaInfo, UiaaResponse}, OutgoingResponse, }, OwnedServerName, @@ -57,6 +49,8 @@ pub enum Error { Path(#[from] axum::extract::rejection::PathRejection), // ruma + #[error("uiaa")] + Uiaa(UiaaInfo), #[error("{0}")] Mxid(#[from] ruma::IdParseError), #[error("{0}: {1}")] @@ -81,8 +75,6 @@ pub enum Error { BadServerResponse(&'static str), #[error("{0}")] Conflict(&'static str), // This is only needed for when a room alias already exists - #[error("uiaa")] - Uiaa(UiaaInfo), // unique / untyped #[error("{0}")] @@ -103,11 +95,10 @@ impl Error { /// Returns the Matrix error code / error kind #[inline] pub fn error_code(&self) -> ruma::api::client::error::ErrorKind { - if let Self::Federation(_, error) = self { - return error.error_kind().unwrap_or_else(|| &Unknown).clone(); - } + use ruma::api::client::error::ErrorKind::Unknown; match self { + Self::Federation(_, err) => err.error_kind().unwrap_or(&Unknown).clone(), Self::BadRequest(kind, _) => kind.clone(), _ => Unknown, } @@ -116,12 +107,8 @@ impl Error { /// Sanitizes public-facing errors that can leak sensitive information. pub fn sanitized_error(&self) -> String { match self { - Self::Database { - .. - } => String::from("Database error occurred."), - Self::Io { - .. - } => String::from("I/O error occurred."), + Self::Database(..) => String::from("Database error occurred."), + Self::Io(..) => String::from("I/O error occurred."), _ => self.to_string(), } } @@ -135,6 +122,88 @@ impl fmt::Debug for Error { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{self}") } } +impl axum::response::IntoResponse for Error { + fn into_response(self) -> axum::response::Response { + let response: UiaaResponse = self.into(); + response.try_into_http_response::().map_or_else( + |_| StatusCode::INTERNAL_SERVER_ERROR.into_response(), + |r| r.map(BytesMut::freeze).map(Full::new).into_response(), + ) + } +} + +impl From for UiaaResponse { + fn from(error: Error) -> Self { + use ruma::api::client::error::{Error as RumaError, ErrorBody, ErrorKind::Unknown}; + + if let Error::Uiaa(uiaainfo) = error { + return Self::AuthResponse(uiaainfo); + } + + let kind = match &error { + Error::Federation(_, ref error) => error.error_kind().unwrap_or(&Unknown), + Error::BadRequest(kind, _) => kind, + _ => &Unknown, + }; + + let status_code = match &error { + Error::Federation(_, ref error) => error.status_code, + Error::BadRequest(ref kind, _) => bad_request_code(kind), + Error::Conflict(_) => StatusCode::CONFLICT, + _ => StatusCode::INTERNAL_SERVER_ERROR, + }; + + let message = if let Error::Federation(ref origin, ref error) = &error { + format!("Answer from {origin}: {error}") + } else { + format!("{error}") + }; + + let body = ErrorBody::Standard { + kind: kind.clone(), + message, + }; + + Self::MatrixError(RumaError { + status_code, + body, + }) + } +} + +fn bad_request_code(kind: &ruma::api::client::error::ErrorKind) -> StatusCode { + use ruma::api::client::error::ErrorKind::*; + + match kind { + GuestAccessForbidden + | ThreepidAuthFailed + | UserDeactivated + | ThreepidDenied + | WrongRoomKeysVersion { + .. + } + | Forbidden { + .. + } => StatusCode::FORBIDDEN, + + UnknownToken { + .. + } + | MissingToken + | Unauthorized => StatusCode::UNAUTHORIZED, + + LimitExceeded { + .. + } => StatusCode::TOO_MANY_REQUESTS, + + TooLarge => StatusCode::PAYLOAD_TOO_LARGE, + + NotFound | Unrecognized => StatusCode::NOT_FOUND, + + _ => StatusCode::BAD_REQUEST, + } +} + #[inline] pub fn log(e: Error) { error!("{e}"); @@ -146,86 +215,3 @@ pub fn debug_log(e: Error) { debug_error!("{e}"); drop(e); } - -#[derive(Clone)] -pub struct RumaResponse(pub T); - -impl From for RumaResponse { - fn from(t: T) -> Self { Self(t) } -} - -impl From for RumaResponse { - fn from(t: Error) -> Self { t.to_response() } -} - -impl Error { - pub fn to_response(&self) -> RumaResponse { - use ruma::api::client::error::{Error as RumaError, ErrorBody}; - - if let Self::Uiaa(uiaainfo) = self { - return RumaResponse(UiaaResponse::AuthResponse(uiaainfo.clone())); - } - - if let Self::Federation(origin, error) = self { - let mut error = error.clone(); - error.body = ErrorBody::Standard { - kind: error.error_kind().unwrap_or_else(|| &Unknown).clone(), - message: format!("Answer from {origin}: {error}"), - }; - return RumaResponse(UiaaResponse::MatrixError(error)); - } - - let message = format!("{self}"); - let (kind, status_code) = match self { - Self::BadRequest(kind, _) => ( - kind.clone(), - match kind { - WrongRoomKeysVersion { - .. - } - | Forbidden { - .. - } - | GuestAccessForbidden - | ThreepidAuthFailed - | UserDeactivated - | ThreepidDenied => StatusCode::FORBIDDEN, - Unauthorized - | UnknownToken { - .. - } - | MissingToken => StatusCode::UNAUTHORIZED, - NotFound | Unrecognized => StatusCode::NOT_FOUND, - LimitExceeded { - .. - } => StatusCode::TOO_MANY_REQUESTS, - TooLarge => StatusCode::PAYLOAD_TOO_LARGE, - _ => StatusCode::BAD_REQUEST, - }, - ), - Self::Conflict(_) => (Unknown, StatusCode::CONFLICT), - _ => (Unknown, StatusCode::INTERNAL_SERVER_ERROR), - }; - - RumaResponse(UiaaResponse::MatrixError(RumaError { - body: ErrorBody::Standard { - kind, - message, - }, - status_code, - })) - } -} - -impl ::axum::response::IntoResponse for Error { - fn into_response(self) -> ::axum::response::Response { self.to_response().into_response() } -} - -impl IntoResponse for RumaResponse { - fn into_response(self) -> Response { - match self.0.try_into_http_response::() { - Ok(res) => res.map(BytesMut::freeze).map(Full::new).into_response(), - Err(_) => StatusCode::INTERNAL_SERVER_ERROR.into_response(), - } - } -} diff --git a/src/core/mod.rs b/src/core/mod.rs index ec536ee2..de8057fa 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -10,7 +10,7 @@ pub mod utils; pub mod version; pub use config::Config; -pub use error::{Error, RumaResponse}; +pub use error::Error; pub use pdu::{PduBuilder, PduCount, PduEvent}; pub use server::Server; pub use version::version; diff --git a/src/router/request.rs b/src/router/request.rs index 079106ef..567de81f 100644 --- a/src/router/request.rs +++ b/src/router/request.rs @@ -1,13 +1,13 @@ use std::sync::{atomic::Ordering, Arc}; use axum::{extract::State, response::IntoResponse}; -use conduit::{debug_error, debug_warn, defer, Result, RumaResponse, Server}; +use conduit::{debug, debug_error, debug_warn, defer, error, trace, Result, Server}; +use conduit_api::RumaResponse; use http::{Method, StatusCode, Uri}; use ruma::api::client::{ error::{Error as RumaError, ErrorBody, ErrorKind}, uiaa::UiaaResponse, }; -use tracing::{debug, error, trace}; #[tracing::instrument(skip_all, level = "debug")] pub(crate) async fn spawn( From a43c78e55f2127cac83a8ebb525cf69eb692b809 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 8 Jul 2024 18:54:42 +0000 Subject: [PATCH 048/158] add RumaError to Error; encapsulate RumaResponse in api Signed-off-by: Jason Volk --- src/api/mod.rs | 3 +-- src/api/router/mod.rs | 3 +-- src/api/router/response.rs | 3 +-- src/core/error.rs | 55 +++++++++++++++++++++++++------------- src/router/request.rs | 16 +++++------ 5 files changed, 45 insertions(+), 35 deletions(-) diff --git a/src/api/mod.rs b/src/api/mod.rs index 0395da7a..6adf2d39 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -9,8 +9,7 @@ extern crate conduit_service as service; pub(crate) use conduit::{debug_info, debug_warn, utils, Error, Result}; pub(crate) use service::{pdu::PduEvent, services, user_is_local}; -pub(crate) use self::router::Ruma; -pub use self::router::RumaResponse; +pub(crate) use self::router::{Ruma, RumaResponse}; conduit::mod_ctor! {} conduit::mod_dtor! {} diff --git a/src/api/router/mod.rs b/src/api/router/mod.rs index f167a606..c3e08c5b 100644 --- a/src/api/router/mod.rs +++ b/src/api/router/mod.rs @@ -13,9 +13,8 @@ use ruma::{ CanonicalJsonValue, OwnedDeviceId, OwnedServerName, OwnedUserId, UserId, }; -pub(super) use self::handler::RouterExt; -pub use self::response::RumaResponse; use self::{auth::Auth, request::Request}; +pub(super) use self::{handler::RouterExt, response::RumaResponse}; use crate::{service::appservice::RegistrationInfo, services, Error, Result}; /// Extractor for Ruma request structs diff --git a/src/api/router/response.rs b/src/api/router/response.rs index 38e58ba9..9b67f37b 100644 --- a/src/api/router/response.rs +++ b/src/api/router/response.rs @@ -5,8 +5,7 @@ use http::StatusCode; use http_body_util::Full; use ruma::api::{client::uiaa::UiaaResponse, OutgoingResponse}; -#[derive(Clone)] -pub struct RumaResponse(pub T); +pub(crate) struct RumaResponse(pub(crate) T); impl From for RumaResponse { fn from(t: Error) -> Self { Self(t.into()) } diff --git a/src/core/error.rs b/src/core/error.rs index cb74d531..5f4d4798 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -4,17 +4,13 @@ use bytes::BytesMut; use http::StatusCode; use http_body_util::Full; use ruma::{ - api::{ - client::uiaa::{UiaaInfo, UiaaResponse}, - OutgoingResponse, - }, + api::{client::uiaa::UiaaResponse, OutgoingResponse}, OwnedServerName, }; -use thiserror::Error; use crate::{debug_error, error}; -#[derive(Error)] +#[derive(thiserror::Error)] pub enum Error { // std #[error("{0}")] @@ -47,10 +43,16 @@ pub enum Error { Extension(#[from] axum::extract::rejection::ExtensionRejection), #[error("{0}")] Path(#[from] axum::extract::rejection::PathRejection), + #[error("{0}")] + Http(#[from] http::Error), // ruma + #[error("{0}")] + IntoHttpError(#[from] ruma::api::error::IntoHttpError), + #[error("{0}")] + RumaError(#[from] ruma::api::client::error::Error), #[error("uiaa")] - Uiaa(UiaaInfo), + Uiaa(ruma::api::client::uiaa::UiaaInfo), #[error("{0}")] Mxid(#[from] ruma::IdParseError), #[error("{0}: {1}")] @@ -98,7 +100,7 @@ impl Error { use ruma::api::client::error::ErrorKind::Unknown; match self { - Self::Federation(_, err) => err.error_kind().unwrap_or(&Unknown).clone(), + Self::Federation(_, error) => ruma_error_kind(error).clone(), Self::BadRequest(kind, _) => kind.clone(), _ => Unknown, } @@ -134,37 +136,35 @@ impl axum::response::IntoResponse for Error { impl From for UiaaResponse { fn from(error: Error) -> Self { - use ruma::api::client::error::{Error as RumaError, ErrorBody, ErrorKind::Unknown}; - if let Error::Uiaa(uiaainfo) = error { return Self::AuthResponse(uiaainfo); } let kind = match &error { - Error::Federation(_, ref error) => error.error_kind().unwrap_or(&Unknown), + Error::Federation(_, ref error) | Error::RumaError(ref error) => ruma_error_kind(error), Error::BadRequest(kind, _) => kind, - _ => &Unknown, + _ => &ruma::api::client::error::ErrorKind::Unknown, }; let status_code = match &error { - Error::Federation(_, ref error) => error.status_code, + Error::Federation(_, ref error) | Error::RumaError(ref error) => error.status_code, Error::BadRequest(ref kind, _) => bad_request_code(kind), Error::Conflict(_) => StatusCode::CONFLICT, _ => StatusCode::INTERNAL_SERVER_ERROR, }; - let message = if let Error::Federation(ref origin, ref error) = &error { - format!("Answer from {origin}: {error}") - } else { - format!("{error}") + let message = match &error { + Error::Federation(ref origin, ref error) => format!("Answer from {origin}: {error}"), + Error::RumaError(ref error) => ruma_error_message(error), + _ => format!("{error}"), }; - let body = ErrorBody::Standard { + let body = ruma::api::client::error::ErrorBody::Standard { kind: kind.clone(), message, }; - Self::MatrixError(RumaError { + Self::MatrixError(ruma::api::client::error::Error { status_code, body, }) @@ -204,6 +204,23 @@ fn bad_request_code(kind: &ruma::api::client::error::ErrorKind) -> StatusCode { } } +fn ruma_error_message(error: &ruma::api::client::error::Error) -> String { + if let ruma::api::client::error::ErrorBody::Standard { + message, + .. + } = &error.body + { + return message.to_string(); + } + + format!("{error}") +} + +fn ruma_error_kind(e: &ruma::api::client::error::Error) -> &ruma::api::client::error::ErrorKind { + e.error_kind() + .unwrap_or(&ruma::api::client::error::ErrorKind::Unknown) +} + #[inline] pub fn log(e: Error) { error!("{e}"); diff --git a/src/router/request.rs b/src/router/request.rs index 567de81f..9256fb9c 100644 --- a/src/router/request.rs +++ b/src/router/request.rs @@ -1,13 +1,9 @@ use std::sync::{atomic::Ordering, Arc}; use axum::{extract::State, response::IntoResponse}; -use conduit::{debug, debug_error, debug_warn, defer, error, trace, Result, Server}; -use conduit_api::RumaResponse; +use conduit::{debug, debug_error, debug_warn, defer, error, trace, Error, Result, Server}; use http::{Method, StatusCode, Uri}; -use ruma::api::client::{ - error::{Error as RumaError, ErrorBody, ErrorKind}, - uiaa::UiaaResponse, -}; +use ruma::api::client::error::{Error as RumaError, ErrorBody, ErrorKind}; #[tracing::instrument(skip_all, level = "debug")] pub(crate) async fn spawn( @@ -66,15 +62,15 @@ fn handle_result( ) -> Result { handle_result_log(method, uri, &result); match result.status() { - StatusCode::METHOD_NOT_ALLOWED => handle_result_403(method, uri, &result), + StatusCode::METHOD_NOT_ALLOWED => handle_result_405(method, uri, &result), _ => Ok(result), } } -fn handle_result_403( +fn handle_result_405( _method: &Method, _uri: &Uri, result: &axum::response::Response, ) -> Result { - let error = UiaaResponse::MatrixError(RumaError { + let error = Error::RumaError(RumaError { status_code: result.status(), body: ErrorBody::Standard { kind: ErrorKind::Unrecognized, @@ -82,7 +78,7 @@ fn handle_result_403( }, }); - Ok(RumaResponse(error).into_response()) + Ok(error.into_response()) } fn handle_result_log(method: &Method, uri: &Uri, result: &axum::response::Response) { From 4718387dbe4c3da072b2652393762f335d5ae888 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 9 Jul 2024 03:31:58 +0000 Subject: [PATCH 049/158] fix wrapper macro semicolons Signed-off-by: Jason Volk --- src/core/debug.rs | 10 +++++----- src/core/log/mod.rs | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/core/debug.rs b/src/core/debug.rs index 7522aa77..1f855e52 100644 --- a/src/core/debug.rs +++ b/src/core/debug.rs @@ -14,9 +14,9 @@ pub use crate::utils::debug::*; macro_rules! debug_event { ( $level:expr, $($x:tt)+ ) => { if cfg!(debug_assertions) && cfg!(not(feature = "dev_release_log_level")) { - ::tracing::event!( $level, $($x)+ ); + ::tracing::event!( $level, $($x)+ ) } else { - ::tracing::debug!( $($x)+ ); + ::tracing::debug!( $($x)+ ) } } } @@ -27,7 +27,7 @@ macro_rules! debug_event { #[macro_export] macro_rules! debug_error { ( $($x:tt)+ ) => { - $crate::debug_event!(::tracing::Level::ERROR, $($x)+ ); + $crate::debug_event!(::tracing::Level::ERROR, $($x)+ ) } } @@ -37,7 +37,7 @@ macro_rules! debug_error { #[macro_export] macro_rules! debug_warn { ( $($x:tt)+ ) => { - $crate::debug_event!(::tracing::Level::WARN, $($x)+ ); + $crate::debug_event!(::tracing::Level::WARN, $($x)+ ) } } @@ -47,7 +47,7 @@ macro_rules! debug_warn { #[macro_export] macro_rules! debug_info { ( $($x:tt)+ ) => { - $crate::debug_event!(::tracing::Level::INFO, $($x)+ ); + $crate::debug_event!(::tracing::Level::INFO, $($x)+ ) } } diff --git a/src/core/log/mod.rs b/src/core/log/mod.rs index daa6b8e8..04d250a6 100644 --- a/src/core/log/mod.rs +++ b/src/core/log/mod.rs @@ -29,25 +29,25 @@ pub struct Log { #[macro_export] macro_rules! error { - ( $($x:tt)+ ) => { ::tracing::error!( $($x)+ ); } + ( $($x:tt)+ ) => { ::tracing::error!( $($x)+ ) } } #[macro_export] macro_rules! warn { - ( $($x:tt)+ ) => { ::tracing::warn!( $($x)+ ); } + ( $($x:tt)+ ) => { ::tracing::warn!( $($x)+ ) } } #[macro_export] macro_rules! info { - ( $($x:tt)+ ) => { ::tracing::info!( $($x)+ ); } + ( $($x:tt)+ ) => { ::tracing::info!( $($x)+ ) } } #[macro_export] macro_rules! debug { - ( $($x:tt)+ ) => { ::tracing::debug!( $($x)+ ); } + ( $($x:tt)+ ) => { ::tracing::debug!( $($x)+ ) } } #[macro_export] macro_rules! trace { - ( $($x:tt)+ ) => { ::tracing::trace!( $($x)+ ); } + ( $($x:tt)+ ) => { ::tracing::trace!( $($x)+ ) } } From 158de10fe6582b2fe3fa3e45c60edfe1626f14c0 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 9 Jul 2024 03:36:08 +0000 Subject: [PATCH 050/158] log erroring errors; improve inspection functors. Signed-off-by: Jason Volk --- src/api/router/response.rs | 13 ++++++---- src/core/error.rs | 49 +++++++++++++++++++++++------------- src/service/admin/console.rs | 2 +- 3 files changed, 40 insertions(+), 24 deletions(-) diff --git a/src/api/router/response.rs b/src/api/router/response.rs index 9b67f37b..2aaa79fa 100644 --- a/src/api/router/response.rs +++ b/src/api/router/response.rs @@ -1,6 +1,6 @@ use axum::response::{IntoResponse, Response}; use bytes::BytesMut; -use conduit::Error; +use conduit::{error, Error}; use http::StatusCode; use http_body_util::Full; use ruma::api::{client::uiaa::UiaaResponse, OutgoingResponse}; @@ -13,9 +13,12 @@ impl From for RumaResponse { impl IntoResponse for RumaResponse { fn into_response(self) -> Response { - match self.0.try_into_http_response::() { - Ok(res) => res.map(BytesMut::freeze).map(Full::new).into_response(), - Err(_) => StatusCode::INTERNAL_SERVER_ERROR.into_response(), - } + self.0 + .try_into_http_response::() + .inspect_err(|e| error!("response error: {e}")) + .map_or_else( + |_| StatusCode::INTERNAL_SERVER_ERROR.into_response(), + |r| r.map(BytesMut::freeze).map(Full::new).into_response(), + ) } } diff --git a/src/core/error.rs b/src/core/error.rs index 5f4d4798..1959081a 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -116,21 +116,46 @@ impl Error { } } -impl From for Error { - fn from(i: Infallible) -> Self { match i {} } +#[inline] +pub fn log(e: &Error) { + error!(?e); +} + +#[inline] +pub fn debug_log(e: &Error) { + debug_error!(?e); +} + +#[inline] +pub fn into_log(e: Error) { + error!(?e); + drop(e); +} + +#[inline] +pub fn into_debug_log(e: Error) { + debug_error!(?e); + drop(e); } impl fmt::Debug for Error { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{self}") } } +impl From for Error { + fn from(i: Infallible) -> Self { match i {} } +} + impl axum::response::IntoResponse for Error { fn into_response(self) -> axum::response::Response { let response: UiaaResponse = self.into(); - response.try_into_http_response::().map_or_else( - |_| StatusCode::INTERNAL_SERVER_ERROR.into_response(), - |r| r.map(BytesMut::freeze).map(Full::new).into_response(), - ) + response + .try_into_http_response::() + .inspect_err(|e| error!(?e)) + .map_or_else( + |_| StatusCode::INTERNAL_SERVER_ERROR.into_response(), + |r| r.map(BytesMut::freeze).map(Full::new).into_response(), + ) } } @@ -220,15 +245,3 @@ fn ruma_error_kind(e: &ruma::api::client::error::Error) -> &ruma::api::client::e e.error_kind() .unwrap_or(&ruma::api::client::error::ErrorKind::Unknown) } - -#[inline] -pub fn log(e: Error) { - error!("{e}"); - drop(e); -} - -#[inline] -pub fn debug_log(e: Error) { - debug_error!("{e}"); - drop(e); -} diff --git a/src/service/admin/console.rs b/src/service/admin/console.rs index 27a4b6c7..2f66b1d5 100644 --- a/src/service/admin/console.rs +++ b/src/service/admin/console.rs @@ -95,7 +95,7 @@ impl Console { ReadlineEvent::Line(string) => self.clone().handle(string).await, ReadlineEvent::Interrupted => continue, ReadlineEvent::Eof => break, - ReadlineEvent::Quit => services().server.shutdown().unwrap_or_else(error::log), + ReadlineEvent::Quit => services().server.shutdown().unwrap_or_else(error::into_log), }, Err(error) => match error { ReadlineError::Closed => break, From 3259ea08b5643a2e362179947420a6dbd1e49d8b Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 5 Jul 2024 15:18:03 -0400 Subject: [PATCH 051/158] use cond compilation for config check, fix docker check w/unix sockets, use our logging instead of tracing Signed-off-by: strawberry --- src/core/config/check.rs | 106 +++++++++++++++++++-------------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/src/core/config/check.rs b/src/core/config/check.rs index 795f0c43..95b4df4e 100644 --- a/src/core/config/check.rs +++ b/src/core/config/check.rs @@ -1,17 +1,12 @@ -#[cfg(unix)] -use std::path::Path; // not unix specific, just only for UNIX sockets stuff and *nix container checks - -use tracing::{debug, error, info, warn}; - -use crate::{error::Error, Config}; +use crate::{error::Error, warn, Config}; pub fn check(config: &Config) -> Result<(), Error> { - #[cfg(feature = "rocksdb")] + #[cfg(all(feature = "rocksdb", not(feature = "sha256_media")))] // prevents catching this in `--all-features` warn!( - "Note the rocksdb feature was deleted from conduwuit, sqlite was deleted and RocksDB is the only supported \ - backend now. Please update your build script to remove this feature." + "Note the rocksdb feature was deleted from conduwuit. SQLite support was removed and RocksDB is the only \ + supported backend now. Please update your build script to remove this feature." ); - #[cfg(feature = "sha256_media")] + #[cfg(all(feature = "sha256_media", not(feature = "rocksdb")))] // prevents catching this in `--all-features` warn!( "Note the sha256_media feature was deleted from conduwuit, it is now fully integrated in a \ forwards-compatible way. Please update your build script to remove this feature." @@ -24,72 +19,77 @@ pub fn check(config: &Config) -> Result<(), Error> { return Err(Error::bad_config("Sentry cannot be enabled without an endpoint set")); } - if cfg!(feature = "hardened_malloc") && cfg!(feature = "jemalloc") { - warn!("hardened_malloc and jemalloc are both enabled, this causes jemalloc to be used."); - } + #[cfg(all(feature = "hardened_malloc", feature = "jemalloc"))] + warn!( + "hardened_malloc and jemalloc are both enabled, this causes jemalloc to be used. If using --all-features, \ + this is harmless." + ); - if config.unix_socket_path.is_some() && !cfg!(unix) { + #[cfg(not(unix))] + if config.unix_socket_path.is_some() { return Err(Error::bad_config( "UNIX socket support is only available on *nix platforms. Please remove \"unix_socket_path\" from your \ config.", )); } - config.get_bind_addrs().iter().for_each(|addr| { - if addr.ip().is_loopback() && cfg!(unix) { - debug!("Found loopback listening address {addr}, running checks if we're in a container.",); + #[cfg(unix)] + if config.unix_socket_path.is_none() { + config.get_bind_addrs().iter().for_each(|addr| { + use std::path::Path; - #[cfg(unix)] - if Path::new("/proc/vz").exists() /* Guest */ && !Path::new("/proc/bz").exists() - /* Host */ - { - error!( - "You are detected using OpenVZ with a loopback/localhost listening address of {addr}. If you are \ - using OpenVZ for containers and you use NAT-based networking to communicate with the host and \ - guest, this will NOT work. Please change this to \"0.0.0.0\". If this is expected, you can \ - ignore.", - ); - } + if addr.ip().is_loopback() { + crate::debug_info!("Found loopback listening address {addr}, running checks if we're in a container.",); - #[cfg(unix)] - if Path::new("/.dockerenv").exists() { - error!( - "You are detected using Docker with a loopback/localhost listening address of {addr}. If you are \ - using a reverse proxy on the host and require communication to conduwuit in the Docker container \ - via NAT-based networking, this will NOT work. Please change this to \"0.0.0.0\". If this is \ - expected, you can ignore.", - ); - } + if Path::new("/proc/vz").exists() /* Guest */ && !Path::new("/proc/bz").exists() + /* Host */ + { + crate::error!( + "You are detected using OpenVZ with a loopback/localhost listening address of {addr}. If you \ + are using OpenVZ for containers and you use NAT-based networking to communicate with the \ + host and guest, this will NOT work. Please change this to \"0.0.0.0\". If this is expected, \ + you can ignore.", + ); + } - #[cfg(unix)] - if Path::new("/run/.containerenv").exists() { - error!( - "You are detected using Podman with a loopback/localhost listening address of {addr}. If you are \ - using a reverse proxy on the host and require communication to conduwuit in the Podman container \ - via NAT-based networking, this will NOT work. Please change this to \"0.0.0.0\". If this is \ - expected, you can ignore.", - ); + if Path::new("/.dockerenv").exists() { + crate::error!( + "You are detected using Docker with a loopback/localhost listening address of {addr}. If you \ + are using a reverse proxy on the host and require communication to conduwuit in the Docker \ + container via NAT-based networking, this will NOT work. Please change this to \"0.0.0.0\". \ + If this is expected, you can ignore.", + ); + } + + if Path::new("/run/.containerenv").exists() { + crate::error!( + "You are detected using Podman with a loopback/localhost listening address of {addr}. If you \ + are using a reverse proxy on the host and require communication to conduwuit in the Podman \ + container via NAT-based networking, this will NOT work. Please change this to \"0.0.0.0\". \ + If this is expected, you can ignore.", + ); + } } - } - }); + }); + } // rocksdb does not allow max_log_files to be 0 if config.rocksdb_max_log_files == 0 { return Err(Error::bad_config( - "When using RocksDB, rocksdb_max_log_files cannot be 0. Please set a value at least 1.", + "rocksdb_max_log_files cannot be 0. Please set a value at least 1.", )); } // yeah, unless the user built a debug build hopefully for local testing only - if config.server_name == "your.server.name" && !cfg!(debug_assertions) { + #[cfg(not(debug_assertions))] + if config.server_name == "your.server.name" { return Err(Error::bad_config( "You must specify a valid server name for production usage of conduwuit.", )); } - if cfg!(debug_assertions) { - info!("Note: conduwuit was built without optimisations (i.e. debug build)"); - } + #[cfg(debug_assertions)] + crate::info!("Note: conduwuit was built without optimisations (i.e. debug build)"); // check if the user specified a registration token as `""` if config.registration_token == Some(String::new()) { @@ -103,7 +103,7 @@ pub fn check(config: &Config) -> Result<(), Error> { // check if user specified valid IP CIDR ranges on startup for cidr in &config.ip_range_denylist { if let Err(e) = ipaddress::IPAddress::parse(cidr) { - error!("Error parsing specified IP CIDR range from string: {e}"); + crate::error!("Error parsing specified IP CIDR range from string: {e}"); return Err(Error::bad_config("Error parsing specified IP CIDR ranges from strings")); } } From daa5c34ea354523446eaca948d2dcfbcd75b3187 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 5 Jul 2024 15:41:16 -0400 Subject: [PATCH 052/158] fix empty version string for NixOS users Signed-off-by: strawberry --- src/core/version.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/core/version.rs b/src/core/version.rs index bd89bee2..2876cea8 100644 --- a/src/core/version.rs +++ b/src/core/version.rs @@ -27,5 +27,11 @@ fn init_user_agent() -> String { format!("{}/{}", name(), version()) } fn init_version() -> String { option_env!("CONDUWUIT_VERSION_EXTRA") .or(option_env!("CONDUIT_VERSION_EXTRA")) - .map_or(SEMANTIC.to_owned(), |extra| format!("{SEMANTIC} ({extra})")) + .map_or(SEMANTIC.to_owned(), |extra| { + if extra.is_empty() { + SEMANTIC.to_owned() + } else { + format!("{SEMANTIC} ({extra})") + } + }) } From e54f4d43978b5eee5356b390144a89fecfb4121d Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 5 Jul 2024 15:50:49 -0400 Subject: [PATCH 053/158] ci: move complement diff results test output Signed-off-by: strawberry --- .github/workflows/ci.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f01e865d..71d8d2a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -164,10 +164,6 @@ jobs: - name: Diff Complement results with checked-in repo results run: | diff -u --color=always tests/test_results/complement/test_results.jsonl complement_test_results.jsonl > >(tee -a complement_test_output.log) - echo '# Complement diff results' >> $GITHUB_STEP_SUMMARY - echo '```diff' >> $GITHUB_STEP_SUMMARY - tail -n 100 complement_test_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - name: Update Job Summary if: success() || failure() @@ -175,9 +171,15 @@ jobs: if [ ${{ job.status }} == 'success' ]; then echo '# ✅ completed suwuccessfully' >> $GITHUB_STEP_SUMMARY else + echo '# CI failure' >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY tail -n 40 test_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY + + echo '# Complement diff results' >> $GITHUB_STEP_SUMMARY + echo '```diff' >> $GITHUB_STEP_SUMMARY + tail -n 100 complement_test_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY fi build: From da03de1d322dbca83683e4baf9b1cff20f1da584 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 5 Jul 2024 16:28:23 -0400 Subject: [PATCH 054/158] bump flake.lock fully MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit • Updated input 'crane': 'github:ipetkov/crane/109987da061a1bf452f435f1653c47511587d919' (2024-05-24) → 'github:ipetkov/crane/087e08a41009bf083d51ab35d8e30b1b7eafa7b0' (2024-07-03) • Updated input 'fenix': 'github:nix-community/fenix/b6fc5035b28e36a98370d0eac44f4ef3fd323df6' (2024-05-22) → 'github:nix-community/fenix/f6994934e25396d3a70ddb908cefccd8d3c37ac4' (2024-07-05) • Updated input 'fenix/rust-analyzer-src': 'github:rust-lang/rust-analyzer/21ec8f523812b88418b2bfc64240c62b3dd967bd' (2024-05-19) → 'github:rust-lang/rust-analyzer/cae997e3380363a906588f14c7b4587f39cf09f5' (2024-07-03) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/5710852ba686cc1fd0d3b8e22b3117d43ba374c2' (2024-05-21) → 'github:NixOS/nixpkgs/9f4128e00b0ae8ec65918efeba59db998750ead6' (2024-07-03) Signed-off-by: strawberry --- flake.lock | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/flake.lock b/flake.lock index 1a064510..06a4cea1 100644 --- a/flake.lock +++ b/flake.lock @@ -123,11 +123,11 @@ ] }, "locked": { - "lastModified": 1716569590, - "narHash": "sha256-5eDbq8TuXFGGO3mqJFzhUbt5zHVTf5zilQoyW5jnJwo=", + "lastModified": 1720025378, + "narHash": "sha256-zlIdj0oLvMEHlllP/7tvY+kE987xsN4FPux6WHSOh00=", "owner": "ipetkov", "repo": "crane", - "rev": "109987da061a1bf452f435f1653c47511587d919", + "rev": "087e08a41009bf083d51ab35d8e30b1b7eafa7b0", "type": "github" }, "original": { @@ -209,11 +209,11 @@ "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1716359173, - "narHash": "sha256-pYcjP6Gy7i6jPWrjiWAVV0BCQp+DdmGaI/k65lBb/kM=", + "lastModified": 1720160983, + "narHash": "sha256-GPl5qug68zcCBvkakakTdzuA/LIOdWyJbAjXkoeM+FE=", "owner": "nix-community", "repo": "fenix", - "rev": "b6fc5035b28e36a98370d0eac44f4ef3fd323df6", + "rev": "f6994934e25396d3a70ddb908cefccd8d3c37ac4", "type": "github" }, "original": { @@ -606,11 +606,11 @@ }, "nixpkgs_4": { "locked": { - "lastModified": 1716330097, - "narHash": "sha256-8BO3B7e3BiyIDsaKA0tY8O88rClYRTjvAp66y+VBUeU=", + "lastModified": 1720031269, + "narHash": "sha256-rwz8NJZV+387rnWpTYcXaRNvzUSnnF9aHONoJIYmiUQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "5710852ba686cc1fd0d3b8e22b3117d43ba374c2", + "rev": "9f4128e00b0ae8ec65918efeba59db998750ead6", "type": "github" }, "original": { @@ -705,11 +705,11 @@ "rust-analyzer-src": { "flake": false, "locked": { - "lastModified": 1716107283, - "narHash": "sha256-NJgrwLiLGHDrCia5AeIvZUHUY7xYGVryee0/9D3Ir1I=", + "lastModified": 1719997308, + "narHash": "sha256-dQx1p/2ObV+iDriPWTBvELCwxe9ZbOimKTJKE3MA2FQ=", "owner": "rust-lang", "repo": "rust-analyzer", - "rev": "21ec8f523812b88418b2bfc64240c62b3dd967bd", + "rev": "cae997e3380363a906588f14c7b4587f39cf09f5", "type": "github" }, "original": { From 391bfd986e00f07063e74deac2fe38947c75d207 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 5 Jul 2024 16:29:19 -0400 Subject: [PATCH 055/158] use ruma's X-Matrix to construct the sending X-Matrix header value Signed-off-by: strawberry --- src/service/sending/send.rs | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/service/sending/send.rs b/src/service/sending/send.rs index a89ea2f8..f4825ff3 100644 --- a/src/service/sending/send.rs +++ b/src/service/sending/send.rs @@ -8,6 +8,8 @@ use ruma::{ client::error::Error as RumaError, EndpointError, IncomingResponse, MatrixVersion, OutgoingRequest, SendAccessToken, }, + serde::Base64, + server_util::authorization::XMatrix, ServerName, }; use tracing::{debug, trace}; @@ -196,16 +198,20 @@ where for signature_server in signatures { for s in signature_server { + let key = + s.0.as_str() + .try_into() + .expect("valid homeserver signing key ID"); + let sig = Base64::parse(s.1).expect("valid base64"); + http_request.headers_mut().insert( AUTHORIZATION, - HeaderValue::from_str(&format!( - "X-Matrix origin=\"{}\",destination=\"{}\",key=\"{}\",sig=\"{}\"", - services().globals.config.server_name, - dest, - s.0, - s.1 - )) - .expect("formatted X-Matrix header"), + HeaderValue::from(&XMatrix::new( + services().globals.config.server_name.clone(), + dest.to_owned(), + key, + sig, + )), ); } } From 60141950f746467e1de5f3153909ff8f82e8a26d Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 5 Jul 2024 16:31:54 -0400 Subject: [PATCH 056/158] fix wrong deactivate-all argument comment Signed-off-by: strawberry --- src/admin/user/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/admin/user/mod.rs b/src/admin/user/mod.rs index 31bf57d6..088133a5 100644 --- a/src/admin/user/mod.rs +++ b/src/admin/user/mod.rs @@ -49,7 +49,7 @@ pub(super) enum UserCommand { /// Markdown code block below the command. DeactivateAll { #[arg(short, long)] - /// Remove users from their joined rooms + /// Does not leave any rooms the user is in on deactivation no_leave_rooms: bool, #[arg(short, long)] /// Also deactivate admin accounts and will assume leave all rooms too From 1c453b1b5511406e554f6827e4e124aaeb17e8b5 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 5 Jul 2024 16:36:05 -0400 Subject: [PATCH 057/158] require authentication on change password and deactivation routes Signed-off-by: strawberry --- src/api/client/account.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/api/client/account.rs b/src/api/client/account.rs index 71324c1b..0d8d6075 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -423,7 +423,12 @@ pub(crate) async fn register_route( pub(crate) async fn change_password_route( InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + // Authentication for this endpoint was made optional, but we need + // authentication currently + let sender_user = body + .sender_user + .as_ref() + .ok_or_else(|| Error::BadRequest(ErrorKind::MissingToken, "Missing access token."))?; let sender_device = body.sender_device.as_ref().expect("user is authenticated"); let mut uiaainfo = UiaaInfo { @@ -512,7 +517,12 @@ pub(crate) async fn whoami_route(body: Ruma) -> Result, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + // Authentication for this endpoint was made optional, but we need + // authentication currently + let sender_user = body + .sender_user + .as_ref() + .ok_or_else(|| Error::BadRequest(ErrorKind::MissingToken, "Missing access token."))?; let sender_device = body.sender_device.as_ref().expect("user is authenticated"); let mut uiaainfo = UiaaInfo { From 83e853e7a341a7b10c8e9bf32e78db21ced1830d Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 5 Jul 2024 16:39:57 -0400 Subject: [PATCH 058/158] add "unstable" endpoint legacy Element Android/iOS seems to call Signed-off-by: strawberry --- src/api/client/thirdparty.rs | 12 +++++++++++- src/api/routes.rs | 2 ++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/api/client/thirdparty.rs b/src/api/client/thirdparty.rs index de24c0ec..f6af8729 100644 --- a/src/api/client/thirdparty.rs +++ b/src/api/client/thirdparty.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; use ruma::api::client::thirdparty::get_protocols; -use crate::{Result, Ruma}; +use crate::{Result, Ruma, RumaResponse}; /// # `GET /_matrix/client/r0/thirdparty/protocols` /// @@ -15,3 +15,13 @@ pub(crate) async fn get_protocols_route( protocols: BTreeMap::new(), }) } + +/// # `GET /_matrix/client/unstable/thirdparty/protocols` +/// +/// Same as `get_protocols_route`, except for some reason Element Android legacy +/// calls this +pub(crate) async fn get_protocols_route_unstable( + body: Ruma, +) -> Result> { + get_protocols_route(body).await.map(RumaResponse) +} diff --git a/src/api/routes.rs b/src/api/routes.rs index 94951aec..b22a32cb 100644 --- a/src/api/routes.rs +++ b/src/api/routes.rs @@ -94,6 +94,8 @@ pub fn build(router: Router, server: &Server) -> Router { .ruma_route(client::search_users_route) .ruma_route(client::get_member_events_route) .ruma_route(client::get_protocols_route) + .route("/_matrix/client/unstable/thirdparty/protocols", + get(client::get_protocols_route_unstable)) .ruma_route(client::send_message_event_route) .ruma_route(client::send_state_event_for_key_route) .ruma_route(client::get_state_events_route) From 68ad351f84b3edcadf364f1dc3c2ba6110a6acf9 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 5 Jul 2024 20:13:21 -0400 Subject: [PATCH 059/158] nix: partially remove some liburing overrides the argument one is weird and causing build script invalid arguments Signed-off-by: strawberry --- flake.nix | 10 ---------- nix/pkgs/main/default.nix | 5 ++--- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/flake.nix b/flake.nix index 499d828d..b252193e 100644 --- a/flake.nix +++ b/flake.nix @@ -50,16 +50,6 @@ configureFlags = pkgs.lib.subtractLists [ "--enable-static" "--disable-shared" ] old.configureFlags; - - postInstall = old.postInstall + '' - # we remove the extra outputs - # - # we need to do this to prevent rocksdb from trying to link the - # static library in a dynamic stdenv - rm $out/lib/liburing*${ - if pkgs.stdenv.hostPlatform.isStatic then ".so*" else ".a" - } - ''; }); }); diff --git a/nix/pkgs/main/default.nix b/nix/pkgs/main/default.nix index f91a9cdd..5c8ade13 100644 --- a/nix/pkgs/main/default.nix +++ b/nix/pkgs/main/default.nix @@ -43,7 +43,7 @@ features'' = lib.subtractLists disable_features' features'; featureEnabled = feature : builtins.elem feature features''; -enableLiburing = featureEnabled "io_uring" && stdenv.isLinux; +enableLiburing = featureEnabled "io_uring" && !stdenv.isDarwin; # This derivation will set the JEMALLOC_OVERRIDE variable, causing the # tikv-jemalloc-sys crate to use the nixpkgs jemalloc instead of building it's @@ -74,8 +74,7 @@ buildDepsOnlyEnv = # TODO: static rocksdb fails to build on darwin # build log at meta.broken = stdenv.hostPlatform.isStatic && stdenv.isDarwin; - # TODO: switch to enableUring option once https://github.com/NixOS/nixpkgs/pull/314945 is available - buildInputs = old.buildInputs ++ lib.optional enableLiburing liburing; + enableLiburing = enableLiburing; }); in { From 373991a8d6952185109e722031563c2b7f1e1f92 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 5 Jul 2024 21:11:54 -0400 Subject: [PATCH 060/158] cleanup and fix backfill from server getting Signed-off-by: strawberry --- src/api/client/account.rs | 2 +- src/service/rooms/timeline/mod.rs | 90 +++++++++++-------------------- 2 files changed, 33 insertions(+), 59 deletions(-) diff --git a/src/api/client/account.rs b/src/api/client/account.rs index 0d8d6075..66dc7a26 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -309,7 +309,7 @@ pub(crate) async fn register_route( // log in conduit admin channel if a guest registered if body.appservice_info.is_none() && is_guest && services().globals.log_guest_registrations() { - info!("New guest user \"{user_id}\" registered on this server from IP."); + info!("New guest user \"{user_id}\" registered on this server."); if let Some(device_display_name) = &body.initial_device_display_name { if body diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index db7af004..96f137e8 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -9,7 +9,6 @@ use std::{ use conduit::{debug, error, info, utils, utils::mutex_map, validated, warn, Error, Result}; use data::Data; use itertools::Itertools; -use rand::prelude::SliceRandom; use ruma::{ api::{client::error::ErrorKind, federation}, canonical_json::to_canonical_value, @@ -1081,7 +1080,7 @@ impl Service { Ok(()) } - #[tracing::instrument(skip(self, room_id))] + #[tracing::instrument(skip(self))] pub async fn backfill_if_required(&self, room_id: &RoomId, from: PduCount) -> Result<()> { let first_pdu = self .all_pdus(user_id!("@doesntmatter:conduit.rs"), room_id)? @@ -1093,41 +1092,6 @@ impl Service { return Ok(()); } - let mut servers: Vec = vec![]; - - // add server names of any trusted key servers if they're in the room - servers.extend( - services() - .rooms - .state_cache - .room_servers(room_id) - .filter_map(Result::ok) - .filter(|server_name| { - services().globals.trusted_servers().contains(server_name) && !server_is_ours(server_name) - }), - ); - - // add server names from room aliases on the room ID - let room_aliases = services() - .rooms - .alias - .local_aliases_for_room(room_id) - .collect::, _>>(); - if let Ok(aliases) = &room_aliases { - for alias in aliases { - if !server_is_ours(alias.server_name()) { - servers.push(alias.server_name().to_owned()); - } - } - } - - // add room ID server name for backfill server - if let Some(server_name) = room_id.server_name() { - if !server_is_ours(server_name) { - servers.push(server_name.to_owned()); - } - } - let power_levels: RoomPowerLevelsEventContent = services() .rooms .state_accessor @@ -1139,29 +1103,39 @@ impl Service { .transpose()? .unwrap_or_default(); - // add server names of the list of admins in the room for backfill server - servers.extend( - power_levels - .users - .iter() - .filter(|(_, level)| **level > power_levels.users_default) - .map(|(user_id, _)| user_id.server_name()) - .filter(|server_name| !server_is_ours(server_name)) - .map(ToOwned::to_owned), - ); + let room_mods = power_levels.users.iter().filter_map(|(user_id, level)| { + if level > &power_levels.users_default && !server_is_ours(user_id.server_name()) { + Some(user_id.server_name().to_owned()) + } else { + None + } + }); - // don't backfill from ourselves (might be noop if we checked it above already) - if let Some(server_index) = servers - .clone() - .into_iter() - .position(|server_name| server_is_ours(&server_name)) - { - servers.swap_remove(server_index); - } + let room_alias_servers = services() + .rooms + .alias + .local_aliases_for_room(room_id) + .filter_map(|alias| { + alias + .ok() + .filter(|alias| !server_is_ours(alias.server_name())) + .map(|alias| alias.server_name().to_owned()) + }); - servers.sort_unstable(); - servers.dedup(); - servers.shuffle(&mut rand::thread_rng()); + let servers = room_mods + .chain(room_alias_servers) + .chain(services().globals.config.trusted_servers.clone()) + .filter(|server_name| { + if server_is_ours(server_name) { + return false; + } + + services() + .rooms + .state_cache + .server_in_room(server_name, room_id) + .unwrap_or(false) + }); for backfill_server in servers { info!("Asking {backfill_server} for backfill"); From 6abc4ad798db2751bb1a0c235a640f170e69f670 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 6 Jul 2024 14:43:17 -0400 Subject: [PATCH 061/158] make `local_aliases_for_room` db iterator Send Signed-off-by: strawberry --- src/service/rooms/alias/data.rs | 2 +- src/service/rooms/alias/mod.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/service/rooms/alias/data.rs b/src/service/rooms/alias/data.rs index eb99e543..302c21ae 100644 --- a/src/service/rooms/alias/data.rs +++ b/src/service/rooms/alias/data.rs @@ -84,7 +84,7 @@ impl Data { pub(super) fn local_aliases_for_room<'a>( &'a self, room_id: &RoomId, - ) -> Box> + 'a> { + ) -> Box> + 'a + Send> { let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xFF); diff --git a/src/service/rooms/alias/mod.rs b/src/service/rooms/alias/mod.rs index 706bf2f8..4af1035e 100644 --- a/src/service/rooms/alias/mod.rs +++ b/src/service/rooms/alias/mod.rs @@ -97,7 +97,7 @@ impl Service { #[tracing::instrument(skip(self), level = "debug")] pub fn local_aliases_for_room<'a>( &'a self, room_id: &RoomId, - ) -> Box> + 'a> { + ) -> Box> + 'a + Send> { self.db.local_aliases_for_room(room_id) } From 0873e18e14bbcbafa2a92dd881e76f6dd4351df9 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 5 Jul 2024 21:16:41 -0400 Subject: [PATCH 062/158] remove random duplicate function Signed-off-by: strawberry --- src/service/rooms/timeline/mod.rs | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 96f137e8..c82098ba 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -145,29 +145,6 @@ impl Service { } */ - /// Returns the version of a room, if known - /// - /// TODO: use this? - #[allow(dead_code)] - pub fn get_room_version(&self, room_id: &RoomId) -> Result> { - let create_event = services() - .rooms - .state_accessor - .room_state_get(room_id, &StateEventType::RoomCreate, "")?; - - let create_event_content: Option = create_event - .as_ref() - .map(|create_event| { - serde_json::from_str(create_event.content.get()).map_err(|e| { - warn!("Invalid create event: {}", e); - Error::bad_database("Invalid create event in db.") - }) - }) - .transpose()?; - - Ok(create_event_content.map(|content| content.room_version)) - } - /// Returns the json of a pdu. pub fn get_pdu_json(&self, event_id: &EventId) -> Result> { self.db.get_pdu_json(event_id) @@ -188,9 +165,6 @@ impl Service { /// Returns the pdu. /// /// Checks the `eventid_outlierpdu` Tree if not found in the timeline. - /// - /// TODO: use this? - #[allow(dead_code)] #[inline] pub fn get_non_outlier_pdu(&self, event_id: &EventId) -> Result> { self.db.get_non_outlier_pdu(event_id) From b5ee15a216e48a53b5b469d6350901fcffe530f4 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 5 Jul 2024 21:19:27 -0400 Subject: [PATCH 063/158] dont skip_all tracing instrument on fetch_state Signed-off-by: strawberry --- src/service/rooms/event_handler/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 0f7919dd..53ce6f8d 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -953,7 +953,7 @@ impl Service { /// Call /state_ids to find out what the state at this pdu is. We trust the /// server's response to some extend (sic), but we still do a lot of checks /// on the events - #[tracing::instrument(skip_all)] + #[tracing::instrument(skip(self, pub_key_map, create_event, room_version_id))] async fn fetch_state( &self, origin: &ServerName, create_event: &PduEvent, room_id: &RoomId, room_version_id: &RoomVersionId, pub_key_map: &RwLock>>, event_id: &EventId, From efea13a675178e052966c21e079dfa57b8560b26 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 5 Jul 2024 22:05:52 -0400 Subject: [PATCH 064/158] add access control checks for room directory publishing/removing Signed-off-by: strawberry --- src/api/client/directory.rs | 40 +++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index 512dface..7d2aff0d 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -10,10 +10,13 @@ use ruma::{ }, directory::{Filter, PublicRoomJoinRule, PublicRoomsChunk, RoomNetwork}, events::{ - room::join_rules::{JoinRule, RoomJoinRulesEventContent}, + room::{ + join_rules::{JoinRule, RoomJoinRulesEventContent}, + power_levels::{RoomPowerLevels, RoomPowerLevelsEventContent}, + }, StateEventType, }, - uint, ServerName, UInt, + uint, RoomId, ServerName, UInt, UserId, }; use tracing::{error, info, warn}; @@ -103,8 +106,6 @@ pub(crate) async fn get_public_rooms_route( /// # `PUT /_matrix/client/r0/directory/list/room/{roomId}` /// /// Sets the visibility of a given room in the room directory. -/// -/// - TODO: Access control checks #[tracing::instrument(skip_all, fields(%client), name = "room_directory")] pub(crate) async fn set_room_visibility_route( InsecureClientIp(client): InsecureClientIp, body: Ruma, @@ -116,6 +117,8 @@ pub(crate) async fn set_room_visibility_route( return Err(Error::BadRequest(ErrorKind::NotFound, "Room not found")); } + user_can_publish_room(sender_user, &body.room_id)?; + match &body.visibility { room::Visibility::Public => { if services().globals.config.lockdown_public_room_directory && !services().users.is_admin(sender_user)? { @@ -351,3 +354,32 @@ pub(crate) async fn get_public_rooms_filtered_helper( total_room_count_estimate: Some(total_room_count_estimate), }) } + +/// Check whether the user can publish to the room directory via power levels of +/// room history visibility event or room creator +fn user_can_publish_room(user_id: &UserId, room_id: &RoomId) -> Result { + if let Some(event) = + services() + .rooms + .state_accessor + .room_state_get(room_id, &StateEventType::RoomPowerLevels, "")? + { + serde_json::from_str(event.content.get()) + .map_err(|_| Error::bad_database("Invalid event content for m.room.power_levels")) + .map(|content: RoomPowerLevelsEventContent| { + RoomPowerLevels::from(content).user_can_send_state(user_id, StateEventType::RoomHistoryVisibility) + }) + } else if let Some(event) = + services() + .rooms + .state_accessor + .room_state_get(room_id, &StateEventType::RoomCreate, "")? + { + Ok(event.sender == user_id) + } else { + return Err(Error::BadRequest( + ErrorKind::Unauthorized, + "You are not allowed to publish this room to the room directory", + )); + } +} From 35336eb68698f6e5fdd2775ca3e3097a30ba074c Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 6 Jul 2024 21:15:44 -0400 Subject: [PATCH 065/158] ci: use `$COMPLEMENT_SRC` from nix devshell for a pinned complement rev Signed-off-by: strawberry --- .github/workflows/ci.yml | 13 ++++--------- bin/complement | 17 ++++++++++++----- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 71d8d2a2..4c6b3bb6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -129,15 +129,10 @@ jobs: run: | direnv exec . engage > >(tee -a test_output.log) - - name: Sync Complement repository - uses: actions/checkout@v4 - with: - repository: 'matrix-org/complement' - path: complement_src - - name: Run Complement tests run: | - direnv exec . bin/complement 'complement_src' 'complement_test_logs.jsonl' 'complement_test_results.jsonl' + # the nix devshell sets $COMPLEMENT_SRC, so "/dev/null" is no-op + direnv exec . bin/complement "/dev/null" complement_test_logs.jsonl complement_test_results.jsonl > >(tee -a test_output.log) cp -v -f result complement_oci_image.tar.gz - name: Upload Complement OCI image @@ -163,7 +158,7 @@ jobs: - name: Diff Complement results with checked-in repo results run: | - diff -u --color=always tests/test_results/complement/test_results.jsonl complement_test_results.jsonl > >(tee -a complement_test_output.log) + diff -u --color=always tests/test_results/complement/test_results.jsonl complement_test_results.jsonl > >(tee -a complement_diff_output.log) - name: Update Job Summary if: success() || failure() @@ -178,7 +173,7 @@ jobs: echo '# Complement diff results' >> $GITHUB_STEP_SUMMARY echo '```diff' >> $GITHUB_STEP_SUMMARY - tail -n 100 complement_test_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY + tail -n 100 complement_diff_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY fi diff --git a/bin/complement b/bin/complement index c679035a..a715b14b 100755 --- a/bin/complement +++ b/bin/complement @@ -7,7 +7,7 @@ set -euo pipefail # The `COMPLEMENT_SRC` environment variable is set in the Nix dev shell, which # points to a store path containing the Complement source code. It's likely you # want to just pass that as the first argument to use it here. -COMPLEMENT_SRC="$1" +COMPLEMENT_SRC="${COMPLEMENT_SRC:-$1}" # A `.jsonl` file to write test logs to LOG_FILE="$2" @@ -17,12 +17,19 @@ RESULTS_FILE="$3" OCI_IMAGE="complement-conduit:main" -# Complement tests that are skipped due to flakiness/reliability issues (likely -# Complement itself induced based on various open issues) -# -# According to Go docs, these are separated by forward slashes and not pipes (why) +# Complement tests that are skipped due to flakiness/reliability issues SKIPPED_COMPLEMENT_TESTS='-skip=TestClientSpacesSummary.*|TestJoinFederatedRoomFromApplicationServiceBridgeUser.*|TestJumpToDateEndpoint.*' +# $COMPLEMENT_SRC needs to be a directory to Complement source code +if [ -f "$COMPLEMENT_SRC" ]; then + echo "\$COMPLEMENT_SRC must be a directory/path to Complement source code" + exit 1 +fi + +# quick test to make sure we can actually write to $LOG_FILE and $RESULTS_FILE +touch $LOG_FILE && rm -v $LOG_FILE +touch $RESULTS_FILE && rm -v $RESULTS_FILE + toplevel="$(git rev-parse --show-toplevel)" pushd "$toplevel" > /dev/null From 0dae9280d9c0046063fa97b18c5979ce0ae42558 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 6 Jul 2024 21:19:52 -0400 Subject: [PATCH 066/158] nix: bump flake.lock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit • Updated input 'crane': 'github:ipetkov/crane/087e08a41009bf083d51ab35d8e30b1b7eafa7b0' (2024-07-03) → 'github:ipetkov/crane/0aed560c5c0a61c9385bddff471a13036203e11c' (2024-07-06) • Updated input 'fenix': 'github:nix-community/fenix/f6994934e25396d3a70ddb908cefccd8d3c37ac4' (2024-07-05) → 'github:nix-community/fenix/27cbad7cc093c5298231b87daa04db9610053651' (2024-07-06) • Updated input 'fenix/rust-analyzer-src': 'github:rust-lang/rust-analyzer/cae997e3380363a906588f14c7b4587f39cf09f5' (2024-07-03) → 'github:rust-lang/rust-analyzer/f2afcb874e7410121c366ae601660abe327e320b' (2024-07-05) • Updated input 'fenix': 'github:nix-community/fenix/27cbad7cc093c5298231b87daa04db9610053651' (2024-07-06) → 'github:nix-community/fenix/abc0549e3560189462a7d394cc9d50af4608d103' (2024-07-08) • Updated input 'fenix/rust-analyzer-src': 'github:rust-lang/rust-analyzer/f2afcb874e7410121c366ae601660abe327e320b' (2024-07-05) → 'github:rust-lang/rust-analyzer/a5b21ea0aa644dffd7cf958b43f11f221d53404e' (2024-07-07) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/9f4128e00b0ae8ec65918efeba59db998750ead6' (2024-07-03) → 'github:NixOS/nixpkgs/655a58a72a6601292512670343087c2d75d859c1' (2024-07-08) Signed-off-by: strawberry --- flake.lock | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/flake.lock b/flake.lock index 06a4cea1..4e6721b2 100644 --- a/flake.lock +++ b/flake.lock @@ -123,11 +123,11 @@ ] }, "locked": { - "lastModified": 1720025378, - "narHash": "sha256-zlIdj0oLvMEHlllP/7tvY+kE987xsN4FPux6WHSOh00=", + "lastModified": 1720226507, + "narHash": "sha256-yHVvNsgrpyNTXZBEokL8uyB2J6gB1wEx0KOJzoeZi1A=", "owner": "ipetkov", "repo": "crane", - "rev": "087e08a41009bf083d51ab35d8e30b1b7eafa7b0", + "rev": "0aed560c5c0a61c9385bddff471a13036203e11c", "type": "github" }, "original": { @@ -209,11 +209,11 @@ "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1720160983, - "narHash": "sha256-GPl5qug68zcCBvkakakTdzuA/LIOdWyJbAjXkoeM+FE=", + "lastModified": 1720420198, + "narHash": "sha256-OIuDb6pHDyGpo7YMFyuRzMLcHm7mRvlYOz0Ht7ps2sU=", "owner": "nix-community", "repo": "fenix", - "rev": "f6994934e25396d3a70ddb908cefccd8d3c37ac4", + "rev": "abc0549e3560189462a7d394cc9d50af4608d103", "type": "github" }, "original": { @@ -606,11 +606,11 @@ }, "nixpkgs_4": { "locked": { - "lastModified": 1720031269, - "narHash": "sha256-rwz8NJZV+387rnWpTYcXaRNvzUSnnF9aHONoJIYmiUQ=", + "lastModified": 1720418205, + "narHash": "sha256-cPJoFPXU44GlhWg4pUk9oUPqurPlCFZ11ZQPk21GTPU=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "9f4128e00b0ae8ec65918efeba59db998750ead6", + "rev": "655a58a72a6601292512670343087c2d75d859c1", "type": "github" }, "original": { @@ -705,11 +705,11 @@ "rust-analyzer-src": { "flake": false, "locked": { - "lastModified": 1719997308, - "narHash": "sha256-dQx1p/2ObV+iDriPWTBvELCwxe9ZbOimKTJKE3MA2FQ=", + "lastModified": 1720344064, + "narHash": "sha256-STmaV9Zu74QtkGGrbr9uMhskwagfCjJqOAYapXabiuk=", "owner": "rust-lang", "repo": "rust-analyzer", - "rev": "cae997e3380363a906588f14c7b4587f39cf09f5", + "rev": "a5b21ea0aa644dffd7cf958b43f11f221d53404e", "type": "github" }, "original": { From d2facaee0b65f5e965651b9c4bbab3b716c01317 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 6 Jul 2024 21:20:51 -0400 Subject: [PATCH 067/158] bump various dependencies Signed-off-by: strawberry --- .github/workflows/trivy.yml | 4 +- .gitlab-ci.yml | 4 +- Cargo.lock | 216 ++++++++++++++++++------------------ Cargo.toml | 6 +- 4 files changed, 115 insertions(+), 115 deletions(-) diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index e9cc9a30..e0871324 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -26,7 +26,7 @@ jobs: uses: actions/checkout@v4 - name: Run Trivy code and vulnerability scanner on repo - uses: aquasecurity/trivy-action@0.23.0 + uses: aquasecurity/trivy-action@0.24.0 with: scan-type: repo format: sarif @@ -34,7 +34,7 @@ jobs: severity: CRITICAL,HIGH,MEDIUM,LOW - name: Run Trivy code and vulnerability scanner on filesystem - uses: aquasecurity/trivy-action@0.23.0 + uses: aquasecurity/trivy-action@0.24.0 with: scan-type: fs format: sarif diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 33d0c6ed..78449e36 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -58,7 +58,7 @@ before_script: ci: stage: ci - image: nixos/nix:2.23.1 + image: nixos/nix:2.23.3 script: # Cache CI dependencies - ./bin/nix-build-and-cache ci @@ -83,7 +83,7 @@ ci: artifacts: stage: artifacts - image: nixos/nix:2.23.1 + image: nixos/nix:2.23.3 script: - ./bin/nix-build-and-cache just .#static-x86_64-unknown-linux-musl - cp result/bin/conduit x86_64-unknown-linux-musl diff --git a/Cargo.lock b/Cargo.lock index dd0c0436..eb24aa21 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -118,18 +118,18 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] name = "async-trait" -version = "0.1.80" +version = "0.1.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" +checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -166,7 +166,7 @@ dependencies = [ "futures-util", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.29", + "hyper 0.14.30", "itoa", "matchit", "memchr", @@ -194,7 +194,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.0", "http-body-util", - "hyper 1.4.0", + "hyper 1.4.1", "hyper-util", "itoa", "matchit", @@ -298,7 +298,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.0", "http-body-util", - "hyper 1.4.0", + "hyper 1.4.1", "hyper-util", "pin-project-lite", "rustls 0.21.12", @@ -377,7 +377,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -474,9 +474,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.104" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74b6a57f98764a267ff415d50a25e6e166f3831a5071af4995296ea97d210490" +checksum = "eaff6f8ce506b9773fa786672d63fc7a191ffea1be33f72bbd4aeacefca9ffc8" dependencies = [ "jobserver", "libc", @@ -535,9 +535,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.8" +version = "4.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84b3edb18336f4df585bc9aa31dd99c036dfa5dc5e9a2939a722a188f3a8970d" +checksum = "64acc1846d54c1fe936a78dc189c34e28d3f5afc348403f28ecf53660b9b8462" dependencies = [ "clap_builder", "clap_derive", @@ -545,9 +545,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.8" +version = "4.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1c09dd5ada6c6c78075d6fd0da3f90d8080651e2d6cc8eb2f1aaa4034ced708" +checksum = "6fb8393d67ba2e7bfaf28a23458e4e2b543cc73a99595511eb207fdb8aede942" dependencies = [ "anstyle", "clap_lex", @@ -562,7 +562,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -638,7 +638,7 @@ dependencies = [ "hmac", "http 1.1.0", "http-body-util", - "hyper 1.4.0", + "hyper 1.4.1", "image", "ipaddress", "jsonwebtoken", @@ -721,7 +721,7 @@ dependencies = [ "conduit_service", "http 1.1.0", "http-body-util", - "hyper 1.4.0", + "hyper 1.4.1", "hyper-util", "log", "ruma", @@ -1007,7 +1007,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -1111,7 +1111,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -1247,7 +1247,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -1528,7 +1528,7 @@ dependencies = [ "markup5ever", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -1616,9 +1616,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.29" +version = "0.14.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f361cde2f109281a220d4307746cdfd5ee3f410da58a70377762396775634b33" +checksum = "a152ddd61dfaec7273fe8419ab357f33aee0d914c5f4efbf0d96fa749eea5ec9" dependencies = [ "bytes", "futures-channel", @@ -1640,9 +1640,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4fe55fb7a772d59a5ff1dfbff4fe0258d19b89fec4b233e75d35d5d2316badc" +checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" dependencies = [ "bytes", "futures-channel", @@ -1667,9 +1667,9 @@ checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" dependencies = [ "futures-util", "http 1.1.0", - "hyper 1.4.0", + "hyper 1.4.1", "hyper-util", - "rustls 0.23.10", + "rustls 0.23.11", "rustls-native-certs", "rustls-pki-types", "tokio", @@ -1684,7 +1684,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper 0.14.29", + "hyper 0.14.30", "pin-project-lite", "tokio", "tokio-io-timeout", @@ -1701,7 +1701,7 @@ dependencies = [ "futures-util", "http 1.1.0", "http-body 1.0.0", - "hyper 1.4.0", + "hyper 1.4.1", "pin-project-lite", "socket2", "tokio", @@ -1937,7 +1937,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -1965,7 +1965,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e310b3a6b5907f99202fcdb4960ff45b93735d7c7d96b760fcff8db2dc0e103d" dependencies = [ "cfg-if", - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -2406,7 +2406,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -2446,7 +2446,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -2539,7 +2539,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -2627,7 +2627,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", "version_check", "yansi", ] @@ -2652,7 +2652,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -2699,7 +2699,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls 0.23.10", + "rustls 0.23.11", "thiserror", "tokio", "tracing", @@ -2715,7 +2715,7 @@ dependencies = [ "rand", "ring", "rustc-hash", - "rustls 0.23.10", + "rustls 0.23.11", "slab", "thiserror", "tinyvec", @@ -2844,7 +2844,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.0", "http-body-util", - "hyper 1.4.0", + "hyper 1.4.1", "hyper-rustls", "hyper-util", "ipnet", @@ -2855,7 +2855,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.10", + "rustls 0.23.11", "rustls-native-certs", "rustls-pemfile", "rustls-pki-types", @@ -3061,7 +3061,7 @@ dependencies = [ "quote", "ruma-identifiers-validation 0.9.5 (git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5)", "serde", - "syn 2.0.68", + "syn 2.0.70", "toml", ] @@ -3197,30 +3197,31 @@ dependencies = [ "log", "ring", "rustls-pki-types", - "rustls-webpki 0.102.4", + "rustls-webpki 0.102.5", "subtle", "zeroize", ] [[package]] name = "rustls" -version = "0.23.10" +version = "0.23.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05cff451f60db80f490f3c182b77c35260baace73209e9cdbbe526bfe3a4d402" +checksum = "4828ea528154ae444e5a642dbb7d5623354030dc9822b83fd9bb79683c7399d0" dependencies = [ + "log", "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.102.4", + "rustls-webpki 0.102.5", "subtle", "zeroize", ] [[package]] name = "rustls-native-certs" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f1fb85efa936c42c6d5fc28d2629bb51e4b2f4b8a5211e297d599cc5a093792" +checksum = "a88d6d420651b496bdd98684116959239430022a115c1240e6c3993be0b15fba" dependencies = [ "openssl-probe", "rustls-pemfile", @@ -3257,9 +3258,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.102.4" +version = "0.102.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e" +checksum = "f9a6fccd794a42c2c105b513a2f62bc3fd8f3ba57a4593677ceb0bd035164d78" dependencies = [ "ring", "rustls-pki-types", @@ -3500,22 +3501,22 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.203" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" +checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.203" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" +checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -3808,9 +3809,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.68" +version = "2.0.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9" +checksum = "2f0209b68b3613b093e0ec905354eccaedcfe83b8cb37cbdeae64026c3064c16" dependencies = [ "proc-macro2", "quote", @@ -3883,7 +3884,7 @@ checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -3982,9 +3983,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.6.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c55115c6fbe2d2bef26eb09ad74bde02d8255476fc0c7b515ef09fbb35742d82" +checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" dependencies = [ "tinyvec_macros", ] @@ -4032,7 +4033,7 @@ checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -4062,7 +4063,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.23.10", + "rustls 0.23.11", "rustls-pki-types", "tokio", ] @@ -4112,7 +4113,7 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.14", + "toml_edit 0.22.15", ] [[package]] @@ -4137,9 +4138,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.14" +version = "0.22.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f21c7aaf97f1bd9ca9d4f9e73b0a6c74bd5afef56f2bc931943a6e1c37e04e38" +checksum = "d59a3a72298453f564e2b111fa896f8d07fabb36f51f06d7e875fc5e0b5a3ef1" dependencies = [ "indexmap 2.2.6", "serde", @@ -4162,7 +4163,7 @@ dependencies = [ "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.29", + "hyper 0.14.30", "hyper-timeout", "percent-encoding", "pin-project", @@ -4248,7 +4249,7 @@ source = "git+https://github.com/girlbossceo/tracing?rev=b348dca742af641c47bc390 dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -4417,16 +4418,15 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "ureq" -version = "2.9.7" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d11a831e3c0b56e438a28308e7c810799e3c118417f342d30ecec080105395cd" +checksum = "72139d247e5f97a3eff96229a7ae85ead5328a39efe76f8bf5a06313d505b6ea" dependencies = [ "base64 0.22.1", "log", "once_cell", - "rustls 0.22.4", + "rustls 0.23.11", "rustls-pki-types", - "rustls-webpki 0.102.4", "url", "webpki-roots", ] @@ -4457,9 +4457,9 @@ checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" [[package]] name = "uuid" -version = "1.9.1" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de17fd2f7da591098415cff336e12965a28061ddace43b59cb3c430179c9439" +checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" dependencies = [ "getrandom", "serde", @@ -4519,7 +4519,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", "wasm-bindgen-shared", ] @@ -4553,7 +4553,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4662,7 +4662,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" dependencies = [ "windows-core", - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -4671,7 +4671,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -4689,7 +4689,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -4709,18 +4709,18 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.5", - "windows_aarch64_msvc 0.52.5", - "windows_i686_gnu 0.52.5", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", "windows_i686_gnullvm", - "windows_i686_msvc 0.52.5", - "windows_x86_64_gnu 0.52.5", - "windows_x86_64_gnullvm 0.52.5", - "windows_x86_64_msvc 0.52.5", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] [[package]] @@ -4731,9 +4731,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" @@ -4743,9 +4743,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" @@ -4755,15 +4755,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" @@ -4773,9 +4773,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" @@ -4785,9 +4785,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" @@ -4797,9 +4797,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" @@ -4809,9 +4809,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" @@ -4876,27 +4876,27 @@ checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" [[package]] name = "zstd" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a" +checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "7.1.0" +version = "7.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a" +checksum = "fa556e971e7b568dc775c136fc9de8c779b1c2fc3a63defaafadffdbd3181afa" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.11+zstd.1.5.6" +version = "2.0.12+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75652c55c0b6f3e6f12eb786fe1bc960396bf05a1eb3bf1f3691c3610ac2e6d4" +checksum = "0a4e40c320c3cb459d9a9ff6de98cff88f4751ee9275d140e2be94a2b74e4c13" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index 0d3d59fd..38065bcc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -113,7 +113,7 @@ features = [ ] [workspace.dependencies.serde] -version = "1.0.203" +version = "1.0.204" features = ["rc"] [workspace.dependencies.serde_json] @@ -169,7 +169,7 @@ default-features = false # used for conduit's CLI and admin room command parsing [workspace.dependencies.clap] -version = "4.5.4" +version = "4.5.9" default-features = false features = [ "std", @@ -265,7 +265,7 @@ version = "2.1.1" version = "0.3.1" [workspace.dependencies.async-trait] -version = "0.1.80" +version = "0.1.81" [workspace.dependencies.lru-cache] version = "0.1.2" From 53fa7c3729d5f80a2b1a17660509bbbf6719adc3 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 7 Jul 2024 02:47:07 -0400 Subject: [PATCH 068/158] nix: add `all-features` outputs/packages Signed-off-by: strawberry --- flake.nix | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/flake.nix b/flake.nix index b252193e..ae8533b9 100644 --- a/flake.nix +++ b/flake.nix @@ -114,9 +114,15 @@ { packages = { default = scopeHost.main; + all-features = scopeHost.main.override { all_features = true; }; hmalloc = scopeHost.main.override { features = ["hardened_malloc"]; }; oci-image = scopeHost.oci-image; + oci-image-all-features = scopeHost.oci-image.override { + main = scopeHost.main.override { + all_features = true; + }; + }; oci-image-hmalloc = scopeHost.oci-image.override { main = scopeHost.main.override { features = ["hardened_malloc"]; @@ -151,6 +157,14 @@ value = scopeCrossStatic.main; } + # An output for a statically-linked binary with `--all-features` + { + name = "${binaryName}-all-features"; + value = scopeCrossStatic.main.override { + all_features = true; + }; + } + # An output for a statically-linked binary with hardened_malloc { name = "${binaryName}-hmalloc"; @@ -165,6 +179,16 @@ value = scopeCrossStatic.oci-image; } + # An output for an OCI image based on that binary with `--all-features` + { + name = "oci-image-${crossSystem}-all-features"; + value = scopeCrossStatic.oci-image.override { + main = scopeCrossStatic.main.override { + all_features = true; + }; + }; + } + # An output for an OCI image based on that binary with hardened_malloc { name = "oci-image-${crossSystem}-hmalloc"; From bd71435a2222fe7afe9587003e1956f558b08cd1 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 7 Jul 2024 02:48:44 -0400 Subject: [PATCH 069/158] ci: build static binaries and OCI imgs with `--all-features` Signed-off-by: strawberry --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c6b3bb6..66ecdce8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -246,7 +246,7 @@ jobs: CARGO_DEB_TARGET_TUPLE=$(echo ${{ matrix.target }} | grep -o -E '^([^-]*-){3}[^-]*') SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) - bin/nix-build-and-cache just .#static-${{ matrix.target }} + bin/nix-build-and-cache just .#static-${{ matrix.target }}-all-features mkdir -v -p target/release/ mkdir -v -p target/$CARGO_DEB_TARGET_TUPLE/release/ cp -v -f result/bin/conduit target/release/conduwuit @@ -273,7 +273,7 @@ jobs: - name: Build OCI image ${{ matrix.target }} run: | - bin/nix-build-and-cache just .#oci-image-${{ matrix.target }} + bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features cp -v -f result oci-image-${{ matrix.target }}.tar.gz - name: Upload OCI image ${{ matrix.target }} From 438911c18d4c93c18ddfb645ad9ec881bb2f7d90 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 7 Jul 2024 12:34:28 -0400 Subject: [PATCH 070/158] nix: remove unnecessary dependencies gathering from allFeatures Signed-off-by: strawberry --- nix/pkgs/main/default.nix | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/nix/pkgs/main/default.nix b/nix/pkgs/main/default.nix index 5c8ade13..2897e675 100644 --- a/nix/pkgs/main/default.nix +++ b/nix/pkgs/main/default.nix @@ -25,11 +25,7 @@ let # on the nix side depend on feature values. crateFeatures = path: let manifest = lib.importTOML "${path}/Cargo.toml"; in - lib.remove "default" (lib.attrNames manifest.features) ++ - lib.attrNames - (lib.filterAttrs - (_: dependency: dependency.optional or false) - manifest.dependencies); + lib.remove "default" (lib.attrNames manifest.features); crateDefaultFeatures = path: (lib.importTOML "${path}/Cargo.toml").features.default; allDefaultFeatures = crateDefaultFeatures "${inputs.self}/src/main"; From 7a4bbe2ff665568aa0cfa807e75551baf08f8aec Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 7 Jul 2024 13:38:38 -0400 Subject: [PATCH 071/158] nix: drop hardened_malloc feature from all-features builds this is non-functional and i've spent far too long trying to figure out what the issue is Signed-off-by: strawberry --- flake.nix | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index ae8533b9..8315c6e1 100644 --- a/flake.nix +++ b/flake.nix @@ -114,13 +114,19 @@ { packages = { default = scopeHost.main; - all-features = scopeHost.main.override { all_features = true; }; + all-features = scopeHost.main.override { + all_features = true; + # this is non-functional on nix for some reason + disable_features = ["hardened_malloc"]; + }; hmalloc = scopeHost.main.override { features = ["hardened_malloc"]; }; oci-image = scopeHost.oci-image; oci-image-all-features = scopeHost.oci-image.override { main = scopeHost.main.override { all_features = true; + # this is non-functional on nix for some reason + disable_features = ["hardened_malloc"]; }; }; oci-image-hmalloc = scopeHost.oci-image.override { @@ -162,6 +168,8 @@ name = "${binaryName}-all-features"; value = scopeCrossStatic.main.override { all_features = true; + # this is non-functional on nix for some reason + disable_features = ["hardened_malloc"]; }; } @@ -185,6 +193,8 @@ value = scopeCrossStatic.oci-image.override { main = scopeCrossStatic.main.override { all_features = true; + # this is non-functional on nix for some reason + disable_features = ["hardened_malloc"]; }; }; } @@ -210,7 +220,11 @@ devShells.default = mkDevShell scopeHostStatic; devShells.all-features = mkDevShell (scopeHostStatic.overrideScope (final: prev: { - main = prev.main.override { all_features = true; }; + main = prev.main.override { + all_features = true; + # this is non-functional on nix for some reason + disable_features = ["hardened_malloc"]; + }; })); devShells.no-features = mkDevShell (scopeHostStatic.overrideScope (final: prev: { From d036d8adcbaa74eee50290a1655be2c2746813e7 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 7 Jul 2024 14:20:35 -0400 Subject: [PATCH 072/158] bump rust-rocksdb to 0.27.1 Signed-off-by: strawberry --- Cargo.lock | 8 ++++---- deps/rust-rocksdb/Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eb24aa21..1846c82d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3123,8 +3123,8 @@ dependencies = [ [[package]] name = "rust-librocksdb-sys" -version = "0.23.0+9.3.1" -source = "git+https://github.com/zaidoon1/rust-rocksdb?rev=b4887edfb84771336930855727390edec07d63fa#b4887edfb84771336930855727390edec07d63fa" +version = "0.23.1+9.3.1" +source = "git+https://github.com/zaidoon1/rust-rocksdb?rev=db1ba33c2b78ad228e2525e8902d059c24fc81a1#db1ba33c2b78ad228e2525e8902d059c24fc81a1" dependencies = [ "bindgen", "bzip2-sys", @@ -3140,8 +3140,8 @@ dependencies = [ [[package]] name = "rust-rocksdb" -version = "0.27.0" -source = "git+https://github.com/zaidoon1/rust-rocksdb?rev=b4887edfb84771336930855727390edec07d63fa#b4887edfb84771336930855727390edec07d63fa" +version = "0.27.1" +source = "git+https://github.com/zaidoon1/rust-rocksdb?rev=db1ba33c2b78ad228e2525e8902d059c24fc81a1#db1ba33c2b78ad228e2525e8902d059c24fc81a1" dependencies = [ "libc", "rust-librocksdb-sys", diff --git a/deps/rust-rocksdb/Cargo.toml b/deps/rust-rocksdb/Cargo.toml index 03efc61c..82b36ff2 100644 --- a/deps/rust-rocksdb/Cargo.toml +++ b/deps/rust-rocksdb/Cargo.toml @@ -27,7 +27,7 @@ malloc-usable-size = ["rust-rocksdb/malloc-usable-size"] [dependencies.rust-rocksdb] git = "https://github.com/zaidoon1/rust-rocksdb" -rev = "b4887edfb84771336930855727390edec07d63fa" +rev = "db1ba33c2b78ad228e2525e8902d059c24fc81a1" #branch = "master" default-features = false From 93e7cf461d7702ea4d9d6cd622c29fcc95eb3d0b Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 7 Jul 2024 14:56:18 -0400 Subject: [PATCH 073/158] add client IP logging to media requests Signed-off-by: strawberry --- src/api/client/media.rs | 60 +++++++++++++++++++++++++++++------------ 1 file changed, 43 insertions(+), 17 deletions(-) diff --git a/src/api/client/media.rs b/src/api/client/media.rs index 44978caf..9d3fbabe 100644 --- a/src/api/client/media.rs +++ b/src/api/client/media.rs @@ -2,6 +2,7 @@ use std::{io::Cursor, sync::Arc, time::Duration}; +use axum_client_ip::InsecureClientIp; use conduit::{debug, error, utils::math::ruma_from_usize, warn}; use image::io::Reader as ImgReader; use ipaddress::IPAddress; @@ -64,18 +65,22 @@ pub(crate) async fn get_media_config_v1_route( /// # `GET /_matrix/media/v3/preview_url` /// /// Returns URL preview. +#[tracing::instrument(skip_all, fields(%client), name = "url_preview")] pub(crate) async fn get_media_preview_route( - body: Ruma, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { + let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let url = &body.url; if !url_preview_allowed(url) { + warn!(%sender_user, "URL is not allowed to be previewed: {url}"); return Err(Error::BadRequest(ErrorKind::forbidden(), "URL is not allowed to be previewed")); } match get_url_preview(url).await { Ok(preview) => { let res = serde_json::value::to_raw_value(&preview).map_err(|e| { - error!("Failed to convert UrlPreviewData into a serde json value: {}", e); + error!(%sender_user, "Failed to convert UrlPreviewData into a serde json value: {e}"); Error::BadRequest( ErrorKind::LimitExceeded { retry_after: Some(RetryAfter::Delay(Duration::from_secs(5))), @@ -87,7 +92,7 @@ pub(crate) async fn get_media_preview_route( Ok(get_media_preview::v3::Response::from_raw_value(res)) }, Err(e) => { - warn!("Failed to generate a URL preview: {e}"); + warn!(%sender_user, "Failed to generate a URL preview: {e}"); // there doesn't seem to be an agreed-upon error code in the spec. // the only response codes in the preview_url spec page are 200 and 429. @@ -108,10 +113,13 @@ pub(crate) async fn get_media_preview_route( /// See /// /// Returns URL preview. +#[tracing::instrument(skip_all, fields(%client), name = "url_preview")] pub(crate) async fn get_media_preview_v1_route( - body: Ruma, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result> { - get_media_preview_route(body).await.map(RumaResponse) + get_media_preview_route(InsecureClientIp(client), body) + .await + .map(RumaResponse) } /// # `POST /_matrix/media/v3/upload` @@ -120,8 +128,9 @@ pub(crate) async fn get_media_preview_v1_route( /// /// - Some metadata will be saved in the database /// - Media will be saved in the media/ directory +#[tracing::instrument(skip_all, fields(%client), name = "media_upload")] pub(crate) async fn create_content_route( - body: Ruma, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -167,10 +176,13 @@ pub(crate) async fn create_content_route( /// /// - Some metadata will be saved in the database /// - Media will be saved in the media/ directory +#[tracing::instrument(skip_all, fields(%client), name = "media_upload")] pub(crate) async fn create_content_v1_route( - body: Ruma, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result> { - create_content_route(body).await.map(RumaResponse) + create_content_route(InsecureClientIp(client), body) + .await + .map(RumaResponse) } /// # `GET /_matrix/media/v3/download/{serverName}/{mediaId}` @@ -181,7 +193,10 @@ pub(crate) async fn create_content_v1_route( /// - Only redirects if `allow_redirect` is true /// - Uses client-provided `timeout_ms` if available, else defaults to 20 /// seconds -pub(crate) async fn get_content_route(body: Ruma) -> Result { +#[tracing::instrument(skip_all, fields(%client), name = "media_get")] +pub(crate) async fn get_content_route( + InsecureClientIp(client): InsecureClientIp, body: Ruma, +) -> Result { let mxc = format!("mxc://{}/{}", body.server_name, body.media_id); if let Some(FileMeta { @@ -243,10 +258,13 @@ pub(crate) async fn get_content_route(body: Ruma) -> R /// - Only redirects if `allow_redirect` is true /// - Uses client-provided `timeout_ms` if available, else defaults to 20 /// seconds +#[tracing::instrument(skip_all, fields(%client), name = "media_get")] pub(crate) async fn get_content_v1_route( - body: Ruma, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result> { - get_content_route(body).await.map(RumaResponse) + get_content_route(InsecureClientIp(client), body) + .await + .map(RumaResponse) } /// # `GET /_matrix/media/v3/download/{serverName}/{mediaId}/{fileName}` @@ -257,8 +275,9 @@ pub(crate) async fn get_content_v1_route( /// - Only redirects if `allow_redirect` is true /// - Uses client-provided `timeout_ms` if available, else defaults to 20 /// seconds +#[tracing::instrument(skip_all, fields(%client), name = "media_get")] pub(crate) async fn get_content_as_filename_route( - body: Ruma, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let mxc = format!("mxc://{}/{}", body.server_name, body.media_id); @@ -328,10 +347,13 @@ pub(crate) async fn get_content_as_filename_route( /// - Only redirects if `allow_redirect` is true /// - Uses client-provided `timeout_ms` if available, else defaults to 20 /// seconds +#[tracing::instrument(skip_all, fields(%client), name = "media_get")] pub(crate) async fn get_content_as_filename_v1_route( - body: Ruma, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result> { - get_content_as_filename_route(body).await.map(RumaResponse) + get_content_as_filename_route(InsecureClientIp(client), body) + .await + .map(RumaResponse) } /// # `GET /_matrix/media/v3/thumbnail/{serverName}/{mediaId}` @@ -342,8 +364,9 @@ pub(crate) async fn get_content_as_filename_v1_route( /// - Only redirects if `allow_redirect` is true /// - Uses client-provided `timeout_ms` if available, else defaults to 20 /// seconds +#[tracing::instrument(skip_all, fields(%client), name = "media_thumbnail_get")] pub(crate) async fn get_content_thumbnail_route( - body: Ruma, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let mxc = format!("mxc://{}/{}", body.server_name, body.media_id); @@ -453,10 +476,13 @@ pub(crate) async fn get_content_thumbnail_route( /// - Only redirects if `allow_redirect` is true /// - Uses client-provided `timeout_ms` if available, else defaults to 20 /// seconds +#[tracing::instrument(skip_all, fields(%client), name = "media_thumbnail_get")] pub(crate) async fn get_content_thumbnail_v1_route( - body: Ruma, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result> { - get_content_thumbnail_route(body).await.map(RumaResponse) + get_content_thumbnail_route(InsecureClientIp(client), body) + .await + .map(RumaResponse) } async fn get_remote_content( From 0fa6976d868b3a1f5c146a2c64e5c796c12a6f32 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 7 Jul 2024 15:02:09 -0400 Subject: [PATCH 074/158] add client IP and user logging on join, remove unnecessary Option Signed-off-by: strawberry --- src/admin/user/commands.rs | 2 +- src/api/client/account.rs | 2 +- src/api/client/membership.rs | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 884e1d29..12aa0170 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -95,7 +95,7 @@ pub(super) async fn create( if let Some(room_id_server_name) = room.server_name() { match join_room_by_id_helper( - Some(&user_id), + &user_id, room, Some("Automatically joining this room upon registration".to_owned()), &[room_id_server_name.to_owned(), services().globals.server_name().to_owned()], diff --git a/src/api/client/account.rs b/src/api/client/account.rs index 66dc7a26..d34211bf 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -376,7 +376,7 @@ pub(crate) async fn register_route( if let Some(room_id_server_name) = room.server_name() { if let Err(e) = join_room_by_id_helper( - Some(&user_id), + &user_id, room, Some("Automatically joining this room upon registration".to_owned()), &[room_id_server_name.to_owned(), services().globals.server_name().to_owned()], diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 0792ed89..6ebcd95b 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -202,7 +202,7 @@ pub(crate) async fn join_room_by_id_route( } join_room_by_id_helper( - body.sender_user.as_deref(), + sender_user, &body.room_id, body.reason.clone(), &servers, @@ -301,7 +301,7 @@ pub(crate) async fn join_room_by_id_or_alias_route( }; let join_room_response = join_room_by_id_helper( - Some(sender_user), + sender_user, &room_id, body.reason.clone(), &servers, @@ -653,11 +653,9 @@ pub(crate) async fn joined_members_route( } pub async fn join_room_by_id_helper( - sender_user: Option<&UserId>, room_id: &RoomId, reason: Option, servers: &[OwnedServerName], + sender_user: &UserId, room_id: &RoomId, reason: Option, servers: &[OwnedServerName], third_party_signed: Option<&ThirdPartySigned>, ) -> Result { - let sender_user = sender_user.expect("user is authenticated"); - if matches!(services().rooms.state_cache.is_joined(sender_user, room_id), Ok(true)) { info!("{sender_user} is already joined in {room_id}"); return Ok(join_room_by_id::v3::Response { @@ -679,6 +677,7 @@ pub async fn join_room_by_id_helper( } } +#[tracing::instrument(skip_all, fields(%sender_user, %room_id), name = "join_remote")] async fn join_room_by_id_helper_remote( sender_user: &UserId, room_id: &RoomId, reason: Option, servers: &[OwnedServerName], _third_party_signed: Option<&ThirdPartySigned>, state_lock: mutex_map::Guard<()>, @@ -1014,6 +1013,7 @@ async fn join_room_by_id_helper_remote( Ok(join_room_by_id::v3::Response::new(room_id.to_owned())) } +#[tracing::instrument(skip_all, fields(%sender_user, %room_id), name = "join_local")] async fn join_room_by_id_helper_local( sender_user: &UserId, room_id: &RoomId, reason: Option, servers: &[OwnedServerName], _third_party_signed: Option<&ThirdPartySigned>, state_lock: mutex_map::Guard<()>, From 192c1e08da38b8be621fd39a9a67be85493136c8 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 7 Jul 2024 15:18:07 -0400 Subject: [PATCH 075/158] add `exclude_disabled` and `exclude_banned` room list admin cmd arguments Signed-off-by: strawberry --- src/admin/room/mod.rs | 12 ++++++++++- src/admin/room/room_commands.rs | 36 +++++++++++++++++++++++++++++---- 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/src/admin/room/mod.rs b/src/admin/room/mod.rs index 72e72793..b4fa15bd 100644 --- a/src/admin/room/mod.rs +++ b/src/admin/room/mod.rs @@ -16,6 +16,14 @@ pub(super) enum RoomCommand { /// - List all rooms the server knows about List { page: Option, + + /// Excludes rooms that we have federation disabled with + #[arg(long)] + exclude_disabled: bool, + + /// Excludes rooms that we have banned + #[arg(long)] + exclude_banned: bool, }, #[command(subcommand)] @@ -179,6 +187,8 @@ pub(super) async fn process(command: RoomCommand, body: Vec<&str>) -> Result list(body, page).await?, + exclude_disabled, + exclude_banned, + } => list(body, page, exclude_disabled, exclude_banned).await?, }) } diff --git a/src/admin/room/room_commands.rs b/src/admin/room/room_commands.rs index f64ccf30..d47edce2 100644 --- a/src/admin/room/room_commands.rs +++ b/src/admin/room/room_commands.rs @@ -1,18 +1,46 @@ use std::fmt::Write; -use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomId}; +use ruma::events::room::message::RoomMessageEventContent; use crate::{escape_html, get_room_info, handler::PAGE_SIZE, services, Result}; -pub(super) async fn list(_body: Vec<&str>, page: Option) -> Result { +pub(super) async fn list( + _body: Vec<&str>, page: Option, exclude_disabled: bool, exclude_banned: bool, +) -> Result { // TODO: i know there's a way to do this with clap, but i can't seem to find it let page = page.unwrap_or(1); let mut rooms = services() .rooms .metadata .iter_ids() - .filter_map(Result::ok) - .map(|id: OwnedRoomId| get_room_info(&id)) + .filter_map(|room_id| { + room_id + .ok() + .filter(|room_id| { + if exclude_disabled + && services() + .rooms + .metadata + .is_disabled(room_id) + .unwrap_or(false) + { + return false; + } + + if exclude_banned + && services() + .rooms + .metadata + .is_banned(room_id) + .unwrap_or(false) + { + return false; + } + + true + }) + .map(|room_id| get_room_info(&room_id)) + }) .collect::>(); rooms.sort_by_key(|r| r.1); rooms.reverse(); From a8e690f22b4678c81d08a62841597ab7a2e73955 Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 8 Jul 2024 17:07:31 -0400 Subject: [PATCH 076/158] nix: unset `patches` in rocksdb we have the patch already and i dont want to make it nix exclusive by reverting it in my rocksdb fork Signed-off-by: strawberry --- flake.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flake.nix b/flake.nix index 8315c6e1..9dc60a99 100644 --- a/flake.nix +++ b/flake.nix @@ -42,6 +42,9 @@ "v" (builtins.fromJSON (builtins.readFile ./flake.lock)) .nodes.rocksdb.original.ref; + # we have this already at https://github.com/girlbossceo/rocksdb/commit/a935c0273e1ba44eacf88ce3685a9b9831486155 + # unsetting this so i don't have to revert it and make this nix exclusive + patches = []; }); # TODO: remove once https://github.com/NixOS/nixpkgs/pull/314945 is available liburing = pkgs.liburing.overrideAttrs (old: { From 05befa4ba248f578643b46bd92450ade849b622c Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 8 Jul 2024 17:08:11 -0400 Subject: [PATCH 077/158] nix: make enableLiburing in rocksdb conditional instead of default true for some reason enableLiburing in nixpkgs rocksdb is default true which breaks Darwin entirely Signed-off-by: strawberry --- nix/pkgs/main/default.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nix/pkgs/main/default.nix b/nix/pkgs/main/default.nix index 2897e675..e3d23a43 100644 --- a/nix/pkgs/main/default.nix +++ b/nix/pkgs/main/default.nix @@ -66,6 +66,10 @@ buildDepsOnlyEnv = # # [1]: https://github.com/tikv/jemallocator/blob/ab0676d77e81268cd09b059260c75b38dbef2d51/jemalloc-sys/src/env.rs#L17 enableJemalloc = featureEnabled "jemalloc" && !stdenv.isDarwin; + + # for some reason enableLiburing in nixpkgs rocksdb is default true + # which breaks Darwin entirely + enableLiburing = enableLiburing; }).overrideAttrs (old: { # TODO: static rocksdb fails to build on darwin # build log at From 53223a4d5ff94e63822ceea567bf5ff1612afeb7 Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 8 Jul 2024 19:17:10 -0400 Subject: [PATCH 078/158] remove snappy as a default feature Signed-off-by: strawberry --- Cargo.toml | 1 - deps/rust-rocksdb/Cargo.toml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 38065bcc..fcbe53d5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -311,7 +311,6 @@ package = "rust-rocksdb-uwu" features = [ "multi-threaded-cf", "mt_static", - "snappy", "lz4", "zstd", "zlib", diff --git a/deps/rust-rocksdb/Cargo.toml b/deps/rust-rocksdb/Cargo.toml index 82b36ff2..57b71491 100644 --- a/deps/rust-rocksdb/Cargo.toml +++ b/deps/rust-rocksdb/Cargo.toml @@ -10,7 +10,7 @@ repository.workspace = true version = "0.0.1" [features] -default = ["snappy", "lz4", "zstd", "zlib", "bzip2"] +default = ["lz4", "zstd", "zlib", "bzip2"] jemalloc = ["rust-rocksdb/jemalloc"] io-uring = ["rust-rocksdb/io-uring"] valgrind = ["rust-rocksdb/valgrind"] From f129d90900fe238a114578bb74c0fc219230cfe2 Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 8 Jul 2024 19:51:05 -0400 Subject: [PATCH 079/158] nix: delete `-DWITH_SNAPPY=1` from rocksdb cmakeFlags Signed-off-by: strawberry --- flake.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/flake.nix b/flake.nix index 9dc60a99..e20da52b 100644 --- a/flake.nix +++ b/flake.nix @@ -45,6 +45,10 @@ # we have this already at https://github.com/girlbossceo/rocksdb/commit/a935c0273e1ba44eacf88ce3685a9b9831486155 # unsetting this so i don't have to revert it and make this nix exclusive patches = []; + # no real reason to have snappy, no one uses this + cmakeFlags = pkgs.lib.subtractLists + [ "-DWITH_SNAPPY=1" ] + old.cmakeFlags; }); # TODO: remove once https://github.com/NixOS/nixpkgs/pull/314945 is available liburing = pkgs.liburing.overrideAttrs (old: { From 080975ab0e862a40a6c035c5198244cff8761e1f Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 9 Jul 2024 08:44:24 -0400 Subject: [PATCH 080/158] adjust/update complement conduwuit config.toml Signed-off-by: strawberry --- nix/pkgs/complement/config.toml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/nix/pkgs/complement/config.toml b/nix/pkgs/complement/config.toml index db1f2d81..15f939bb 100644 --- a/nix/pkgs/complement/config.toml +++ b/nix/pkgs/complement/config.toml @@ -5,13 +5,17 @@ allow_guest_registration = true allow_public_room_directory_over_federation = true allow_public_room_directory_without_auth = true allow_registration = true -allow_unstable_room_versions = true -database_backend = "rocksdb" database_path = "/database" -log = "trace" +log = "trace,h2=warn,hyper=warn" port = [8008, 8448] trusted_servers = [] +query_trusted_key_servers_first = false yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse = true +ip_range_denylist = [] +url_preview_domain_contains_allowlist = ["*"] +media_compat_file_link = false +media_statup_check = false +rocksdb_direct_io = false [global.tls] certs = "/certificate.crt" From aa9540af21d7533f39a8b6d44705f79e7e45935f Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 10 Jul 2024 19:56:34 -0400 Subject: [PATCH 081/158] slightly simplify send_state_event route Signed-off-by: strawberry --- src/api/client/state.rs | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/src/api/client/state.rs b/src/api/client/state.rs index 9247b123..c63ddba5 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -36,18 +36,16 @@ pub(crate) async fn send_state_event_for_key_route( ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let event_id = send_state_event_for_key_helper( - sender_user, - &body.room_id, - &body.event_type, - &body.body.body, - body.state_key.clone(), - ) - .await?; - - let event_id = (*event_id).to_owned(); Ok(send_state_event::v3::Response { - event_id, + event_id: send_state_event_for_key_helper( + sender_user, + &body.room_id, + &body.event_type, + &body.body.body, + body.state_key.clone(), + ) + .await? + .into(), }) } From b5d4a1c1b0c1e5cd8e399caa01744d46a8e34ff4 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 10 Jul 2024 19:57:10 -0400 Subject: [PATCH 082/158] drop harmless state event not found message to debug_info Signed-off-by: strawberry --- src/api/client/state.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/client/state.rs b/src/api/client/state.rs index c63ddba5..abff9218 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use conduit::{error, warn}; +use conduit::{debug_info, error}; use ruma::{ api::client::{ error::ErrorKind, @@ -126,7 +126,7 @@ pub(crate) async fn get_state_events_for_key_route( .state_accessor .room_state_get(&body.room_id, &body.event_type, &body.state_key)? .ok_or_else(|| { - warn!("State event {:?} not found in room {:?}", &body.event_type, &body.room_id); + debug_info!("State event {:?} not found in room {:?}", &body.event_type, &body.room_id); Error::BadRequest(ErrorKind::NotFound, "State event not found.") })?; if body From 8077e910f648078da2eb0e6b1d8733165775265d Mon Sep 17 00:00:00 2001 From: AlexPewMaster Date: Sat, 13 Jul 2024 22:59:07 +0200 Subject: [PATCH 083/158] Fix CONDUWUIT_CONFIG variable in docker-compose.yml files (+ remove deprecated compose version) --- docs/deploying/docker-compose.for-traefik.yml | 3 +-- docs/deploying/docker-compose.override.yml | 1 - docs/deploying/docker-compose.with-caddy.yml | 2 +- docs/deploying/docker-compose.with-traefik.yml | 4 +--- docs/deploying/docker-compose.yml | 4 +--- 5 files changed, 4 insertions(+), 10 deletions(-) diff --git a/docs/deploying/docker-compose.for-traefik.yml b/docs/deploying/docker-compose.for-traefik.yml index ec3e720c..d10e5815 100644 --- a/docs/deploying/docker-compose.for-traefik.yml +++ b/docs/deploying/docker-compose.for-traefik.yml @@ -1,5 +1,4 @@ # conduwuit - Behind Traefik Reverse Proxy -version: '2.4' # uses '2.4' for cpuset services: homeserver: @@ -24,7 +23,7 @@ services: CONDUWUIT_TRUSTED_SERVERS: '["matrix.org"]' #CONDUWUIT_LOG: warn,state_res=warn CONDUWUIT_ADDRESS: 0.0.0.0 - #CONDUWUIT_CONFIG: './conduwuit.toml' # Uncomment if you mapped config toml above + #CONDUWUIT_CONFIG: '/etc/conduwuit.toml' # Uncomment if you mapped config toml above #cpuset: "0-4" # Uncomment to limit to specific CPU cores # We need some way to server the client and server .well-known json. The simplest way is to use a nginx container diff --git a/docs/deploying/docker-compose.override.yml b/docs/deploying/docker-compose.override.yml index 2e937e75..23d6a90b 100644 --- a/docs/deploying/docker-compose.override.yml +++ b/docs/deploying/docker-compose.override.yml @@ -1,5 +1,4 @@ # conduwuit - Traefik Reverse Proxy Labels -version: '2.4' # uses '2.4' for cpuset services: homeserver: diff --git a/docs/deploying/docker-compose.with-caddy.yml b/docs/deploying/docker-compose.with-caddy.yml index 41d8856d..899f4d67 100644 --- a/docs/deploying/docker-compose.with-caddy.yml +++ b/docs/deploying/docker-compose.with-caddy.yml @@ -39,7 +39,7 @@ services: CONDUWUIT_TRUSTED_SERVERS: '["matrix.org"]' #CONDUWUIT_LOG: warn,state_res=warn CONDUWUIT_ADDRESS: 0.0.0.0 - #CONDUWUIT_CONFIG: './conduwuit.toml' # Uncomment if you mapped config toml above + #CONDUWUIT_CONFIG: '/etc/conduwuit.toml' # Uncomment if you mapped config toml above networks: - caddy labels: diff --git a/docs/deploying/docker-compose.with-traefik.yml b/docs/deploying/docker-compose.with-traefik.yml index c93f5414..79d20051 100644 --- a/docs/deploying/docker-compose.with-traefik.yml +++ b/docs/deploying/docker-compose.with-traefik.yml @@ -1,5 +1,4 @@ # conduwuit - Behind Traefik Reverse Proxy -version: '2.4' # uses '2.4' for cpuset services: homeserver: @@ -16,7 +15,7 @@ services: CONDUWUIT_SERVER_NAME: your.server.name # EDIT THIS CONDUWUIT_TRUSTED_SERVERS: '["matrix.org"]' CONDUWUIT_ALLOW_REGISTRATION : 'true' - #CONDUWUIT_CONFIG: './conduwuit.toml' # Uncomment if you mapped config toml above + #CONDUWUIT_CONFIG: '/etc/conduwuit.toml' # Uncomment if you mapped config toml above ### Uncomment and change values as desired # CONDUWUIT_ADDRESS: 0.0.0.0 # CONDUWUIT_PORT: 6167 @@ -28,7 +27,6 @@ services: # CONDUWUIT_DATABASE_PATH: /srv/conduwuit/.local/share/conduwuit # CONDUWUIT_WORKERS: 10 # CONDUWUIT_MAX_REQUEST_SIZE: 20000000 # in bytes, ~20 MB - #cpuset: "0-4" # Uncomment to limit to specific CPU cores # We need some way to server the client and server .well-known json. The simplest way is to use a nginx container # to serve those two as static files. If you want to use a different way, delete or comment the below service, here diff --git a/docs/deploying/docker-compose.yml b/docs/deploying/docker-compose.yml index 066c8fe1..bc9f2477 100644 --- a/docs/deploying/docker-compose.yml +++ b/docs/deploying/docker-compose.yml @@ -1,5 +1,4 @@ # conduwuit -version: '2.4' # uses '2.4' for cpuset services: homeserver: @@ -24,8 +23,7 @@ services: CONDUWUIT_TRUSTED_SERVERS: '["matrix.org"]' #CONDUWUIT_LOG: warn,state_res=warn CONDUWUIT_ADDRESS: 0.0.0.0 - #CONDUWUIT_CONFIG: './conduwuit.toml' # Uncomment if you mapped config toml above - #cpuset: "0-4" # Uncomment to limit to specific CPU cores + #CONDUWUIT_CONFIG: '/etc/conduwuit.toml' # Uncomment if you mapped config toml above # ### Uncomment if you want to use your own Element-Web App. ### Note: You need to provide a config.json for Element and you also need a second From 454dd43d4ca8561cf92a6e0ed89457590311dab4 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 9 Jul 2024 06:04:05 +0000 Subject: [PATCH 084/158] fix membership route ABA's Signed-off-by: Jason Volk --- src/api/client/membership.rs | 42 ++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 6ebcd95b..da96acc8 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -7,7 +7,7 @@ use std::{ use axum_client_ip::InsecureClientIp; use conduit::{ - debug, error, info, trace, utils, + debug, debug_warn, error, info, trace, utils, utils::{math::continue_exponential_backoff_secs, mutex_map}, warn, Error, PduEvent, Result, }; @@ -366,6 +366,12 @@ pub(crate) async fn invite_user_route( pub(crate) async fn kick_user_route(body: Ruma) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let state_lock = services() + .globals + .roomid_mutex_state + .lock(&body.room_id) + .await; + let mut event: RoomMemberEventContent = serde_json::from_str( services() .rooms @@ -383,12 +389,6 @@ pub(crate) async fn kick_user_route(body: Ruma) -> Resul event.membership = MembershipState::Leave; event.reason.clone_from(&body.reason); - let state_lock = services() - .globals - .roomid_mutex_state - .lock(&body.room_id) - .await; - services() .rooms .timeline @@ -417,6 +417,12 @@ pub(crate) async fn kick_user_route(body: Ruma) -> Resul pub(crate) async fn ban_user_route(body: Ruma) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let state_lock = services() + .globals + .roomid_mutex_state + .lock(&body.room_id) + .await; + let event = services() .rooms .state_accessor @@ -447,12 +453,6 @@ pub(crate) async fn ban_user_route(body: Ruma) -> Result< }, )?; - let state_lock = services() - .globals - .roomid_mutex_state - .lock(&body.room_id) - .await; - services() .rooms .timeline @@ -481,6 +481,12 @@ pub(crate) async fn ban_user_route(body: Ruma) -> Result< pub(crate) async fn unban_user_route(body: Ruma) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let state_lock = services() + .globals + .roomid_mutex_state + .lock(&body.room_id) + .await; + let mut event: RoomMemberEventContent = serde_json::from_str( services() .rooms @@ -496,12 +502,6 @@ pub(crate) async fn unban_user_route(body: Ruma) -> Res event.reason.clone_from(&body.reason); event.join_authorized_via_users_server = None; - let state_lock = services() - .globals - .roomid_mutex_state - .lock(&body.room_id) - .await; - services() .rooms .timeline @@ -656,6 +656,8 @@ pub async fn join_room_by_id_helper( sender_user: &UserId, room_id: &RoomId, reason: Option, servers: &[OwnedServerName], third_party_signed: Option<&ThirdPartySigned>, ) -> Result { + let state_lock = services().rooms.state.mutex.lock(room_id).await; + if matches!(services().rooms.state_cache.is_joined(sender_user, room_id), Ok(true)) { info!("{sender_user} is already joined in {room_id}"); return Ok(join_room_by_id::v3::Response { @@ -663,8 +665,6 @@ pub async fn join_room_by_id_helper( }); } - let state_lock = services().globals.roomid_mutex_state.lock(room_id).await; - // Ask a remote server if we are not participating in this room if !services() .rooms From c62d653989ffc1d0e6176d5d33fbc44a6f3df4eb Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 9 Jul 2024 06:08:40 +0000 Subject: [PATCH 085/158] tweak some log levels Signed-off-by: Jason Volk --- src/api/client/membership.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index da96acc8..4b664c5b 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -659,7 +659,7 @@ pub async fn join_room_by_id_helper( let state_lock = services().rooms.state.mutex.lock(room_id).await; if matches!(services().rooms.state_cache.is_joined(sender_user, room_id), Ok(true)) { - info!("{sender_user} is already joined in {room_id}"); + debug_warn!("{sender_user} is already joined in {room_id}"); return Ok(join_room_by_id::v3::Response { room_id: room_id.into(), }); @@ -1018,7 +1018,7 @@ async fn join_room_by_id_helper_local( sender_user: &UserId, room_id: &RoomId, reason: Option, servers: &[OwnedServerName], _third_party_signed: Option<&ThirdPartySigned>, state_lock: mutex_map::Guard<()>, ) -> Result { - info!("We can join locally"); + debug!("We can join locally"); let join_rules_event = services() @@ -1118,7 +1118,7 @@ async fn join_room_by_id_helper_local( .iter() .any(|server_name| !server_is_ours(server_name)) { - info!("We couldn't do the join locally, maybe federation can help to satisfy the restricted join requirements"); + warn!("We couldn't do the join locally, maybe federation can help to satisfy the restricted join requirements"); let (make_join_response, remote_server) = make_join_request(sender_user, room_id, servers).await?; let room_version_id = match make_join_response.room_version { From 5dcdafe2070fb362cb0477ec4ce8b6068b9eb147 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 9 Jul 2024 06:09:12 +0000 Subject: [PATCH 086/158] take local join branch when remote join would fail Signed-off-by: Jason Volk --- src/api/client/membership.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 4b664c5b..5c56faa0 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -665,15 +665,17 @@ pub async fn join_room_by_id_helper( }); } - // Ask a remote server if we are not participating in this room - if !services() + if services() .rooms .state_cache .server_in_room(services().globals.server_name(), room_id)? + || servers.is_empty() + || (servers.len() == 1 && server_is_ours(&servers[0])) { - join_room_by_id_helper_remote(sender_user, room_id, reason, servers, third_party_signed, state_lock).await - } else { join_room_by_id_helper_local(sender_user, room_id, reason, servers, third_party_signed, state_lock).await + } else { + // Ask a remote server if we are not participating in this room + join_room_by_id_helper_remote(sender_user, room_id, reason, servers, third_party_signed, state_lock).await } } From 50c2d2b80151c888ebb49153d1e8b0c9dee02c44 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 9 Jul 2024 06:15:45 +0000 Subject: [PATCH 087/158] add command to force join user to room (#136) Signed-off-by: Jason Volk --- src/admin/user/commands.rs | 16 +++++++++++++++- src/admin/user/mod.rs | 12 +++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 12aa0170..218dadb9 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -8,7 +8,7 @@ use ruma::{ tag::{TagEvent, TagEventContent, TagInfo}, RoomAccountDataEventType, }, - OwnedRoomId, OwnedUserId, RoomId, + OwnedRoomId, OwnedRoomOrAliasId, OwnedUserId, RoomId, }; use tracing::{error, info, warn}; @@ -334,6 +334,20 @@ pub(super) async fn list_joined_rooms(_body: Vec<&str>, user_id: String) -> Resu Ok(RoomMessageEventContent::text_html(output_plain, output_html)) } +pub(super) async fn force_join_room( + _body: Vec<&str>, user_id: String, room_id: OwnedRoomOrAliasId, +) -> Result { + let user_id = parse_local_user_id(&user_id)?; + let room_id = services().rooms.alias.resolve(&room_id).await?; + + assert!(service::user_is_local(&user_id), "Parsed user_id must be a local user"); + join_room_by_id_helper(&user_id, &room_id, None, &[], None).await?; + + Ok(RoomMessageEventContent::notice_markdown(format!( + "{user_id} has been joined to {room_id}.", + ))) +} + pub(super) async fn put_room_tag( _body: Vec<&str>, user_id: String, room_id: Box, tag: String, ) -> Result { diff --git a/src/admin/user/mod.rs b/src/admin/user/mod.rs index 088133a5..e7d3d251 100644 --- a/src/admin/user/mod.rs +++ b/src/admin/user/mod.rs @@ -2,7 +2,7 @@ mod commands; use clap::Subcommand; use conduit::Result; -use ruma::{events::room::message::RoomMessageEventContent, RoomId}; +use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomOrAliasId, RoomId}; use self::commands::*; @@ -65,6 +65,12 @@ pub(super) enum UserCommand { user_id: String, }, + /// - Manually join a local user to a room. + ForceJoinRoom { + user_id: String, + room_id: OwnedRoomOrAliasId, + }, + /// - Puts a room tag for the specified user and room ID. /// /// This is primarily useful if you'd like to set your admin room @@ -113,6 +119,10 @@ pub(super) async fn process(command: UserCommand, body: Vec<&str>) -> Result list_joined_rooms(body, user_id).await?, + UserCommand::ForceJoinRoom { + user_id, + room_id, + } => force_join_room(body, user_id, room_id).await?, UserCommand::PutRoomTag { user_id, room_id, From 01b2928d5581a53cc0e38895b90a3226327907fa Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 9 Jul 2024 06:41:29 +0000 Subject: [PATCH 088/158] add make user admin command (#136) Signed-off-by: Jason Volk --- src/admin/user/commands.rs | 15 +++++++++++++++ src/admin/user/mod.rs | 8 ++++++++ 2 files changed, 23 insertions(+) diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 218dadb9..9e4b348b 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -348,6 +348,21 @@ pub(super) async fn force_join_room( ))) } +pub(super) async fn make_user_admin(_body: Vec<&str>, user_id: String) -> Result { + let user_id = parse_local_user_id(&user_id)?; + let displayname = services() + .users + .displayname(&user_id)? + .unwrap_or_else(|| user_id.to_string()); + + assert!(service::user_is_local(&user_id), "Parsed user_id must be a local user"); + service::admin::make_user_admin(&user_id, displayname).await?; + + Ok(RoomMessageEventContent::notice_markdown(format!( + "{user_id} has been granted admin privileges.", + ))) +} + pub(super) async fn put_room_tag( _body: Vec<&str>, user_id: String, room_id: Box, tag: String, ) -> Result { diff --git a/src/admin/user/mod.rs b/src/admin/user/mod.rs index e7d3d251..cdb5fa5e 100644 --- a/src/admin/user/mod.rs +++ b/src/admin/user/mod.rs @@ -71,6 +71,11 @@ pub(super) enum UserCommand { room_id: OwnedRoomOrAliasId, }, + /// - Grant server-admin privileges to a user. + MakeUserAdmin { + user_id: String, + }, + /// - Puts a room tag for the specified user and room ID. /// /// This is primarily useful if you'd like to set your admin room @@ -123,6 +128,9 @@ pub(super) async fn process(command: UserCommand, body: Vec<&str>) -> Result force_join_room(body, user_id, room_id).await?, + UserCommand::MakeUserAdmin { + user_id, + } => make_user_admin(body, user_id).await?, UserCommand::PutRoomTag { user_id, room_id, From 2d251eb19ca734baa0fdafbe8c5d0993ec771be9 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 9 Jul 2024 20:04:43 +0000 Subject: [PATCH 089/158] cleanup on drop for utils::mutex_map. Signed-off-by: Jason Volk --- src/api/client/membership.rs | 12 +++--- src/core/utils/mod.rs | 2 +- src/core/utils/mutex_map.rs | 53 ++++++++++++++++++------- src/core/utils/tests.rs | 53 +++++++++++++++++++++++++ src/service/admin/mod.rs | 6 +-- src/service/globals/mod.rs | 18 ++++++--- src/service/rooms/state/data.rs | 7 ++-- src/service/rooms/state/mod.rs | 13 +++--- src/service/rooms/state_accessor/mod.rs | 10 ++--- src/service/rooms/timeline/mod.rs | 11 ++--- 10 files changed, 131 insertions(+), 54 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 5c56faa0..2ac4c723 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -7,9 +7,8 @@ use std::{ use axum_client_ip::InsecureClientIp; use conduit::{ - debug, debug_warn, error, info, trace, utils, - utils::{math::continue_exponential_backoff_secs, mutex_map}, - warn, Error, PduEvent, Result, + debug, debug_warn, error, info, trace, utils, utils::math::continue_exponential_backoff_secs, warn, Error, + PduEvent, Result, }; use ruma::{ api::{ @@ -36,13 +35,14 @@ use ruma::{ OwnedUserId, RoomId, RoomVersionId, ServerName, UserId, }; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; -use service::sending::convert_to_outgoing_federation_event; use tokio::sync::RwLock; use crate::{ client::{update_avatar_url, update_displayname}, service::{ + globals::RoomMutexGuard, pdu::{gen_event_id_canonical_json, PduBuilder}, + sending::convert_to_outgoing_federation_event, server_is_ours, user_is_local, }, services, Ruma, @@ -682,7 +682,7 @@ pub async fn join_room_by_id_helper( #[tracing::instrument(skip_all, fields(%sender_user, %room_id), name = "join_remote")] async fn join_room_by_id_helper_remote( sender_user: &UserId, room_id: &RoomId, reason: Option, servers: &[OwnedServerName], - _third_party_signed: Option<&ThirdPartySigned>, state_lock: mutex_map::Guard<()>, + _third_party_signed: Option<&ThirdPartySigned>, state_lock: RoomMutexGuard, ) -> Result { info!("Joining {room_id} over federation."); @@ -1018,7 +1018,7 @@ async fn join_room_by_id_helper_remote( #[tracing::instrument(skip_all, fields(%sender_user, %room_id), name = "join_local")] async fn join_room_by_id_helper_local( sender_user: &UserId, room_id: &RoomId, reason: Option, servers: &[OwnedServerName], - _third_party_signed: Option<&ThirdPartySigned>, state_lock: mutex_map::Guard<()>, + _third_party_signed: Option<&ThirdPartySigned>, state_lock: RoomMutexGuard, ) -> Result { debug!("We can join locally"); diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 2b79c3c4..4e5ba480 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -20,7 +20,7 @@ pub use debug::slice_truncated as debug_slice_truncated; pub use hash::calculate_hash; pub use html::Escape as HtmlEscape; pub use json::{deserialize_from_str, to_canonical_object}; -pub use mutex_map::MutexMap; +pub use mutex_map::{Guard as MutexMapGuard, MutexMap}; pub use rand::string as random_string; pub use string::{str_from_bytes, string_from_bytes}; pub use sys::available_parallelism; diff --git a/src/core/utils/mutex_map.rs b/src/core/utils/mutex_map.rs index f102487c..c3c51798 100644 --- a/src/core/utils/mutex_map.rs +++ b/src/core/utils/mutex_map.rs @@ -1,20 +1,22 @@ -use std::{hash::Hash, sync::Arc}; +use std::{fmt::Debug, hash::Hash, sync::Arc}; -type Value = tokio::sync::Mutex; -type ArcMutex = Arc>; -type HashMap = std::collections::HashMap>; -type MapMutex = std::sync::Mutex>; -type Map = MapMutex; +use tokio::sync::OwnedMutexGuard as Omg; /// Map of Mutexes pub struct MutexMap { map: Map, } -pub struct Guard { - _guard: tokio::sync::OwnedMutexGuard, +pub struct Guard { + map: Map, + val: Omg, } +type Map = Arc>; +type MapMutex = std::sync::Mutex>; +type HashMap = std::collections::HashMap>; +type Value = Arc>; + impl MutexMap where Key: Send + Hash + Eq + Clone, @@ -23,28 +25,38 @@ where #[must_use] pub fn new() -> Self { Self { - map: Map::::new(HashMap::::new()), + map: Map::new(MapMutex::new(HashMap::new())), } } - pub async fn lock(&self, k: &K) -> Guard + #[tracing::instrument(skip(self), level = "debug")] + pub async fn lock(&self, k: &K) -> Guard where - K: ?Sized + Send + Sync, + K: ?Sized + Send + Sync + Debug, Key: for<'a> From<&'a K>, { let val = self .map .lock() - .expect("map mutex locked") + .expect("locked") .entry(k.into()) .or_default() .clone(); - let guard = val.lock_owned().await; - Guard:: { - _guard: guard, + Guard:: { + map: Arc::clone(&self.map), + val: val.lock_owned().await, } } + + #[must_use] + pub fn contains(&self, k: &Key) -> bool { self.map.lock().expect("locked").contains_key(k) } + + #[must_use] + pub fn is_empty(&self) -> bool { self.map.lock().expect("locked").is_empty() } + + #[must_use] + pub fn len(&self) -> usize { self.map.lock().expect("locked").len() } } impl Default for MutexMap @@ -54,3 +66,14 @@ where { fn default() -> Self { Self::new() } } + +impl Drop for Guard { + fn drop(&mut self) { + if Arc::strong_count(Omg::mutex(&self.val)) <= 2 { + self.map + .lock() + .expect("locked") + .retain(|_, val| !Arc::ptr_eq(val, Omg::mutex(&self.val)) || Arc::strong_count(val) > 2); + } + } +} diff --git a/src/core/utils/tests.rs b/src/core/utils/tests.rs index add15861..43968947 100644 --- a/src/core/utils/tests.rs +++ b/src/core/utils/tests.rs @@ -81,3 +81,56 @@ fn checked_add_overflow() { let res = checked!(a + 1).expect("overflow"); assert_eq!(res, 0); } + +#[tokio::test] +async fn mutex_map_cleanup() { + use crate::utils::MutexMap; + + let map = MutexMap::::new(); + + let lock = map.lock("foo").await; + assert!(!map.is_empty(), "map must not be empty"); + + drop(lock); + assert!(map.is_empty(), "map must be empty"); +} + +#[tokio::test] +async fn mutex_map_contend() { + use std::sync::Arc; + + use tokio::sync::Barrier; + + use crate::utils::MutexMap; + + let map = Arc::new(MutexMap::::new()); + let seq = Arc::new([Barrier::new(2), Barrier::new(2)]); + let str = "foo".to_owned(); + + let seq_ = seq.clone(); + let map_ = map.clone(); + let str_ = str.clone(); + let join_a = tokio::spawn(async move { + let _lock = map_.lock(&str_).await; + assert!(!map_.is_empty(), "A0 must not be empty"); + seq_[0].wait().await; + assert!(map_.contains(&str_), "A1 must contain key"); + }); + + let seq_ = seq.clone(); + let map_ = map.clone(); + let str_ = str.clone(); + let join_b = tokio::spawn(async move { + let _lock = map_.lock(&str_).await; + assert!(!map_.is_empty(), "B0 must not be empty"); + seq_[1].wait().await; + assert!(map_.contains(&str_), "B1 must contain key"); + }); + + seq[0].wait().await; + assert!(map.contains(&str), "Must contain key"); + seq[1].wait().await; + + tokio::try_join!(join_b, join_a).expect("joined"); + assert!(map.is_empty(), "Must be empty"); +} diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index ca0e551b..e0dd1760 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -9,7 +9,7 @@ use std::{ }; use async_trait::async_trait; -use conduit::{error, utils::mutex_map, Error, Result}; +use conduit::{error, Error, Result}; pub use create::create_admin_room; pub use grant::make_user_admin; use loole::{Receiver, Sender}; @@ -26,7 +26,7 @@ use tokio::{ task::JoinHandle, }; -use crate::{pdu::PduBuilder, services, user_is_local, PduEvent}; +use crate::{globals::RoomMutexGuard, pdu::PduBuilder, services, user_is_local, PduEvent}; const COMMAND_QUEUE_LIMIT: usize = 512; @@ -270,7 +270,7 @@ async fn respond_to_room(content: RoomMessageEventContent, room_id: &RoomId, use } async fn handle_response_error( - e: &Error, room_id: &RoomId, user_id: &UserId, state_lock: &mutex_map::Guard<()>, + e: &Error, room_id: &RoomId, user_id: &UserId, state_lock: &RoomMutexGuard, ) -> Result<()> { error!("Failed to build and append admin room response PDU: \"{e}\""); let error_room_message = RoomMessageEventContent::text_plain(format!( diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 11bfc88c..a5b70835 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -12,7 +12,11 @@ use std::{ time::Instant, }; -use conduit::{error, trace, utils::MutexMap, Config, Result}; +use conduit::{ + error, trace, + utils::{MutexMap, MutexMapGuard}, + Config, Result, +}; use data::Data; use ipaddress::IPAddress; use regex::RegexSet; @@ -27,8 +31,6 @@ use url::Url; use crate::services; -type RateLimitState = (Instant, u32); // Time if last failed try, number of failed tries - pub struct Service { pub db: Data, @@ -43,9 +45,9 @@ pub struct Service { pub bad_event_ratelimiter: Arc>>, pub bad_signature_ratelimiter: Arc, RateLimitState>>>, pub bad_query_ratelimiter: Arc>>, - pub roomid_mutex_insert: MutexMap, - pub roomid_mutex_state: MutexMap, - pub roomid_mutex_federation: MutexMap, + pub roomid_mutex_insert: RoomMutexMap, + pub roomid_mutex_state: RoomMutexMap, + pub roomid_mutex_federation: RoomMutexMap, pub roomid_federationhandletime: RwLock>, pub updates_handle: Mutex>>, pub stateres_mutex: Arc>, @@ -53,6 +55,10 @@ pub struct Service { pub admin_alias: OwnedRoomAliasId, } +pub type RoomMutexMap = MutexMap; +pub type RoomMutexGuard = MutexMapGuard; +type RateLimitState = (Instant, u32); // Time if last failed try, number of failed tries + impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let config = &args.server.config; diff --git a/src/service/rooms/state/data.rs b/src/service/rooms/state/data.rs index aad3bede..b62adf60 100644 --- a/src/service/rooms/state/data.rs +++ b/src/service/rooms/state/data.rs @@ -3,7 +3,8 @@ use std::{collections::HashSet, sync::Arc}; use conduit::{utils, Error, Result}; use database::{Database, Map}; use ruma::{EventId, OwnedEventId, RoomId}; -use utils::mutex_map; + +use crate::globals::RoomMutexGuard; pub(super) struct Data { shorteventid_shortstatehash: Arc, @@ -35,7 +36,7 @@ impl Data { &self, room_id: &RoomId, new_shortstatehash: u64, - _mutex_lock: &mutex_map::Guard<()>, // Take mutex guard to make sure users get the room state mutex + _mutex_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex ) -> Result<()> { self.roomid_shortstatehash .insert(room_id.as_bytes(), &new_shortstatehash.to_be_bytes())?; @@ -68,7 +69,7 @@ impl Data { &self, room_id: &RoomId, event_ids: Vec, - _mutex_lock: &mutex_map::Guard<()>, // Take mutex guard to make sure users get the room state mutex + _mutex_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex ) -> Result<()> { let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xFF); diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index 52ee89d1..b46a9d04 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -5,10 +5,7 @@ use std::{ sync::Arc, }; -use conduit::{ - utils::{calculate_hash, mutex_map}, - warn, Error, Result, -}; +use conduit::{utils::calculate_hash, warn, Error, Result}; use data::Data; use ruma::{ api::client::error::ErrorKind, @@ -22,7 +19,7 @@ use ruma::{ }; use super::state_compressor::CompressedStateEvent; -use crate::{services, PduEvent}; +use crate::{globals::RoomMutexGuard, services, PduEvent}; pub struct Service { db: Data, @@ -46,7 +43,7 @@ impl Service { shortstatehash: u64, statediffnew: Arc>, _statediffremoved: Arc>, - state_lock: &mutex_map::Guard<()>, // Take mutex guard to make sure users get the room state mutex + state_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex ) -> Result<()> { for event_id in statediffnew.iter().filter_map(|new| { services() @@ -318,7 +315,7 @@ impl Service { &self, room_id: &RoomId, shortstatehash: u64, - mutex_lock: &mutex_map::Guard<()>, // Take mutex guard to make sure users get the room state mutex + mutex_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex ) -> Result<()> { self.db.set_room_state(room_id, shortstatehash, mutex_lock) } @@ -358,7 +355,7 @@ impl Service { &self, room_id: &RoomId, event_ids: Vec, - state_lock: &mutex_map::Guard<()>, // Take mutex guard to make sure users get the room state mutex + state_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex ) -> Result<()> { self.db .set_forward_extremities(room_id, event_ids, state_lock) diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index a3567857..35719c15 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -6,11 +6,7 @@ use std::{ sync::{Arc, Mutex as StdMutex, Mutex}, }; -use conduit::{ - error, - utils::{math::usize_from_f64, mutex_map}, - warn, Error, Result, -}; +use conduit::{error, utils::math::usize_from_f64, warn, Error, Result}; use data::Data; use lru_cache::LruCache; use ruma::{ @@ -37,7 +33,7 @@ use ruma::{ }; use serde_json::value::to_raw_value; -use crate::{pdu::PduBuilder, services, PduEvent}; +use crate::{globals::RoomMutexGuard, pdu::PduBuilder, services, PduEvent}; pub struct Service { db: Data, @@ -333,7 +329,7 @@ impl Service { } pub fn user_can_invite( - &self, room_id: &RoomId, sender: &UserId, target_user: &UserId, state_lock: &mutex_map::Guard<()>, + &self, room_id: &RoomId, sender: &UserId, target_user: &UserId, state_lock: &RoomMutexGuard, ) -> Result { let content = to_raw_value(&RoomMemberEventContent::new(MembershipState::Invite)) .expect("Event content always serializes"); diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index c82098ba..9bfc2715 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -6,7 +6,7 @@ use std::{ sync::Arc, }; -use conduit::{debug, error, info, utils, utils::mutex_map, validated, warn, Error, Result}; +use conduit::{debug, error, info, utils, validated, warn, Error, Result}; use data::Data; use itertools::Itertools; use ruma::{ @@ -36,6 +36,7 @@ use tokio::sync::RwLock; use crate::{ admin, appservice::NamespaceRegex, + globals::RoomMutexGuard, pdu::{EventHash, PduBuilder}, rooms::{event_handler::parse_incoming_pdu, state_compressor::CompressedStateEvent}, server_is_ours, services, PduCount, PduEvent, @@ -203,7 +204,7 @@ impl Service { pdu: &PduEvent, mut pdu_json: CanonicalJsonObject, leaves: Vec, - state_lock: &mutex_map::Guard<()>, // Take mutex guard to make sure users get the room state mutex + state_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex ) -> Result> { // Coalesce database writes for the remainder of this scope. let _cork = services().db.cork_and_flush(); @@ -593,7 +594,7 @@ impl Service { pdu_builder: PduBuilder, sender: &UserId, room_id: &RoomId, - _mutex_lock: &mutex_map::Guard<()>, // Take mutex guard to make sure users get the room state mutex + _mutex_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex ) -> Result<(PduEvent, CanonicalJsonObject)> { let PduBuilder { event_type, @@ -780,7 +781,7 @@ impl Service { pdu_builder: PduBuilder, sender: &UserId, room_id: &RoomId, - state_lock: &mutex_map::Guard<()>, // Take mutex guard to make sure users get the room state mutex + state_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex ) -> Result> { let (pdu, pdu_json) = self.create_hash_and_sign_event(pdu_builder, sender, room_id, state_lock)?; if let Some(admin_room) = admin::Service::get_admin_room()? { @@ -963,7 +964,7 @@ impl Service { new_room_leaves: Vec, state_ids_compressed: Arc>, soft_fail: bool, - state_lock: &mutex_map::Guard<()>, // Take mutex guard to make sure users get the room state mutex + state_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex ) -> Result>> { // We append to state before appending the pdu, so we don't have a moment in // time with the pdu without it's state. This is okay because append_pdu can't From 271f720286280876eeb2be1ba8082dbc1d93274e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 9 Jul 2024 21:10:14 +0000 Subject: [PATCH 090/158] move mutex maps out of globals into respective service Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 2 +- src/admin/federation/commands.rs | 5 +-- src/api/client/membership.rs | 26 ++++----------- src/api/client/message.rs | 6 +--- src/api/client/profile.rs | 2 +- src/api/client/redact.rs | 6 +--- src/api/client/room.rs | 14 ++------ src/api/client/state.rs | 2 +- src/api/client/sync.rs | 6 ++-- src/api/server/make_join.rs | 6 +--- src/api/server/make_leave.rs | 6 +--- src/api/server/send.rs | 5 +-- src/api/server/send_join.rs | 5 +-- src/api/server/send_leave.rs | 5 +-- src/service/admin/create.rs | 2 +- src/service/admin/grant.rs | 2 +- src/service/admin/mod.rs | 4 +-- src/service/globals/mod.rs | 20 ++---------- src/service/rooms/event_handler/mod.rs | 43 ++++++++++++++----------- src/service/rooms/state/data.rs | 2 +- src/service/rooms/state/mod.rs | 14 ++++++-- src/service/rooms/state_accessor/mod.rs | 2 +- src/service/rooms/timeline/mod.rs | 29 ++++++++++------- 23 files changed, 93 insertions(+), 121 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 53009566..62741f38 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -570,7 +570,7 @@ pub(super) async fn force_set_room_state_from_server( .state_compressor .save_state(room_id.clone().as_ref(), new_room_state)?; - let state_lock = services().globals.roomid_mutex_state.lock(&room_id).await; + let state_lock = services().rooms.state.mutex.lock(&room_id).await; services() .rooms .state diff --git a/src/admin/federation/commands.rs b/src/admin/federation/commands.rs index 24f4bc23..a97e7582 100644 --- a/src/admin/federation/commands.rs +++ b/src/admin/federation/commands.rs @@ -16,8 +16,9 @@ pub(super) async fn enable_room(_body: Vec<&str>, room_id: Box) -> Resul pub(super) async fn incoming_federation(_body: Vec<&str>) -> Result { let map = services() - .globals - .roomid_federationhandletime + .rooms + .event_handler + .federation_handletime .read() .expect("locked"); let mut msg = format!("Handling {} incoming pdus:\n", map.len()); diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 2ac4c723..1b70fdec 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -40,8 +40,8 @@ use tokio::sync::RwLock; use crate::{ client::{update_avatar_url, update_displayname}, service::{ - globals::RoomMutexGuard, pdu::{gen_event_id_canonical_json, PduBuilder}, + rooms::state::RoomMutexGuard, sending::convert_to_outgoing_federation_event, server_is_ours, user_is_local, }, @@ -366,11 +366,7 @@ pub(crate) async fn invite_user_route( pub(crate) async fn kick_user_route(body: Ruma) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let state_lock = services() - .globals - .roomid_mutex_state - .lock(&body.room_id) - .await; + let state_lock = services().rooms.state.mutex.lock(&body.room_id).await; let mut event: RoomMemberEventContent = serde_json::from_str( services() @@ -417,11 +413,7 @@ pub(crate) async fn kick_user_route(body: Ruma) -> Resul pub(crate) async fn ban_user_route(body: Ruma) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let state_lock = services() - .globals - .roomid_mutex_state - .lock(&body.room_id) - .await; + let state_lock = services().rooms.state.mutex.lock(&body.room_id).await; let event = services() .rooms @@ -481,11 +473,7 @@ pub(crate) async fn ban_user_route(body: Ruma) -> Result< pub(crate) async fn unban_user_route(body: Ruma) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let state_lock = services() - .globals - .roomid_mutex_state - .lock(&body.room_id) - .await; + let state_lock = services().rooms.state.mutex.lock(&body.room_id).await; let mut event: RoomMemberEventContent = serde_json::from_str( services() @@ -1399,7 +1387,7 @@ pub(crate) async fn invite_helper( if !user_is_local(user_id) { let (pdu, pdu_json, invite_room_state) = { - let state_lock = services().globals.roomid_mutex_state.lock(room_id).await; + let state_lock = services().rooms.state.mutex.lock(room_id).await; let content = to_raw_value(&RoomMemberEventContent { avatar_url: services().users.avatar_url(user_id)?, displayname: None, @@ -1511,7 +1499,7 @@ pub(crate) async fn invite_helper( )); } - let state_lock = services().globals.roomid_mutex_state.lock(room_id).await; + let state_lock = services().rooms.state.mutex.lock(room_id).await; services() .rooms @@ -1605,7 +1593,7 @@ pub async fn leave_room(user_id: &UserId, room_id: &RoomId, reason: Option, user_id: OwnedUserId) { for (pdu_builder, room_id) in all_joined_rooms { - let state_lock = services().globals.roomid_mutex_state.lock(room_id).await; + let state_lock = services().rooms.state.mutex.lock(room_id).await; if let Err(e) = services() .rooms .timeline diff --git a/src/api/client/redact.rs b/src/api/client/redact.rs index 4cb24c33..308d12e5 100644 --- a/src/api/client/redact.rs +++ b/src/api/client/redact.rs @@ -15,11 +15,7 @@ pub(crate) async fn redact_event_route(body: Ruma) -> let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let body = body.body; - let state_lock = services() - .globals - .roomid_mutex_state - .lock(&body.room_id) - .await; + let state_lock = services().rooms.state.mutex.lock(&body.room_id).await; let event_id = services() .rooms diff --git a/src/api/client/room.rs b/src/api/client/room.rs index 7090fdc8..ccdf4dc0 100644 --- a/src/api/client/room.rs +++ b/src/api/client/room.rs @@ -90,7 +90,7 @@ pub(crate) async fn create_room_route(body: Ruma) -> R } let _short_id = services().rooms.short.get_or_create_shortroomid(&room_id)?; - let state_lock = services().globals.roomid_mutex_state.lock(&room_id).await; + let state_lock = services().rooms.state.mutex.lock(&room_id).await; let alias: Option = if let Some(alias) = &body.room_alias_name { Some(room_alias_check(alias, &body.appservice_info).await?) @@ -573,11 +573,7 @@ pub(crate) async fn upgrade_room_route(body: Ruma) -> .short .get_or_create_shortroomid(&replacement_room)?; - let state_lock = services() - .globals - .roomid_mutex_state - .lock(&body.room_id) - .await; + let state_lock = services().rooms.state.mutex.lock(&body.room_id).await; // Send a m.room.tombstone event to the old room to indicate that it is not // intended to be used any further Fail if the sender does not have the required @@ -605,11 +601,7 @@ pub(crate) async fn upgrade_room_route(body: Ruma) -> // Change lock to replacement room drop(state_lock); - let state_lock = services() - .globals - .roomid_mutex_state - .lock(&replacement_room) - .await; + let state_lock = services().rooms.state.mutex.lock(&replacement_room).await; // Get the old room creation event let mut create_event_content = serde_json::from_str::( diff --git a/src/api/client/state.rs b/src/api/client/state.rs index abff9218..25b77fe3 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -170,7 +170,7 @@ async fn send_state_event_for_key_helper( sender: &UserId, room_id: &RoomId, event_type: &StateEventType, json: &Raw, state_key: String, ) -> Result> { allowed_to_send_state_event(room_id, event_type, json).await?; - let state_lock = services().globals.roomid_mutex_state.lock(room_id).await; + let state_lock = services().rooms.state.mutex.lock(room_id).await; let event_id = services() .rooms .timeline diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index 45a1c75b..dd1f3401 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -199,7 +199,7 @@ pub(crate) async fn sync_events_route( let (room_id, invite_state_events) = result?; // Get and drop the lock to wait for remaining operations to finish - let insert_lock = services().globals.roomid_mutex_insert.lock(&room_id).await; + let insert_lock = services().rooms.timeline.mutex_insert.lock(&room_id).await; drop(insert_lock); let invite_count = services() @@ -317,7 +317,7 @@ async fn handle_left_room( next_batch_string: &str, full_state: bool, lazy_load_enabled: bool, ) -> Result<()> { // Get and drop the lock to wait for remaining operations to finish - let insert_lock = services().globals.roomid_mutex_insert.lock(room_id).await; + let insert_lock = services().rooms.timeline.mutex_insert.lock(room_id).await; drop(insert_lock); let left_count = services() @@ -519,7 +519,7 @@ async fn load_joined_room( ) -> Result { // Get and drop the lock to wait for remaining operations to finish // This will make sure the we have all events until next_batch - let insert_lock = services().globals.roomid_mutex_insert.lock(room_id).await; + let insert_lock = services().rooms.timeline.mutex_insert.lock(room_id).await; drop(insert_lock); let (timeline_pdus, limited) = load_timeline(sender_user, room_id, sincecount, 10)?; diff --git a/src/api/server/make_join.rs b/src/api/server/make_join.rs index ca50dcbe..c909ea33 100644 --- a/src/api/server/make_join.rs +++ b/src/api/server/make_join.rs @@ -71,11 +71,7 @@ pub(crate) async fn create_join_event_template_route( let room_version_id = services().rooms.state.get_room_version(&body.room_id)?; - let state_lock = services() - .globals - .roomid_mutex_state - .lock(&body.room_id) - .await; + let state_lock = services().rooms.state.mutex.lock(&body.room_id).await; let join_authorized_via_users_server = if (services() .rooms diff --git a/src/api/server/make_leave.rs b/src/api/server/make_leave.rs index 62c09717..eea3e4f8 100644 --- a/src/api/server/make_leave.rs +++ b/src/api/server/make_leave.rs @@ -35,11 +35,7 @@ pub(crate) async fn create_leave_event_template_route( .acl_check(origin, &body.room_id)?; let room_version_id = services().rooms.state.get_room_version(&body.room_id)?; - let state_lock = services() - .globals - .roomid_mutex_state - .lock(&body.room_id) - .await; + let state_lock = services().rooms.state.mutex.lock(&body.room_id).await; let content = to_raw_value(&RoomMemberEventContent { avatar_url: None, blurhash: None, diff --git a/src/api/server/send.rs b/src/api/server/send.rs index 90225a14..08caf1b4 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -127,8 +127,9 @@ async fn handle_pdus( for (event_id, value, room_id) in parsed_pdus { let pdu_start_time = Instant::now(); let mutex_lock = services() - .globals - .roomid_mutex_federation + .rooms + .event_handler + .mutex_federation .lock(&room_id) .await; resolved_map.insert( diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index ff362f64..577833d5 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -156,8 +156,9 @@ async fn create_join_event( .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "origin is not a server name."))?; let mutex_lock = services() - .globals - .roomid_mutex_federation + .rooms + .event_handler + .mutex_federation .lock(room_id) .await; let pdu_id: Vec = services() diff --git a/src/api/server/send_leave.rs b/src/api/server/send_leave.rs index 4fdde515..c4e17bbc 100644 --- a/src/api/server/send_leave.rs +++ b/src/api/server/send_leave.rs @@ -152,8 +152,9 @@ async fn create_leave_event(origin: &ServerName, room_id: &RoomId, pdu: &RawJson .await?; let mutex_lock = services() - .globals - .roomid_mutex_federation + .rooms + .event_handler + .mutex_federation .lock(room_id) .await; let pdu_id: Vec = services() diff --git a/src/service/admin/create.rs b/src/service/admin/create.rs index ad70fe0c..57df344a 100644 --- a/src/service/admin/create.rs +++ b/src/service/admin/create.rs @@ -34,7 +34,7 @@ pub async fn create_admin_room() -> Result<()> { let _short_id = services().rooms.short.get_or_create_shortroomid(&room_id)?; - let state_lock = services().globals.roomid_mutex_state.lock(&room_id).await; + let state_lock = services().rooms.state.mutex.lock(&room_id).await; // Create a user for the server let server_user = &services().globals.server_user; diff --git a/src/service/admin/grant.rs b/src/service/admin/grant.rs index ca48ce0d..9a4ef242 100644 --- a/src/service/admin/grant.rs +++ b/src/service/admin/grant.rs @@ -22,7 +22,7 @@ use crate::{pdu::PduBuilder, services}; /// In conduit, this is equivalent to granting admin privileges. pub async fn make_user_admin(user_id: &UserId, displayname: String) -> Result<()> { if let Some(room_id) = Service::get_admin_room()? { - let state_lock = services().globals.roomid_mutex_state.lock(&room_id).await; + let state_lock = services().rooms.state.mutex.lock(&room_id).await; // Use the server user to grant the new admin's power level let server_user = &services().globals.server_user; diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index e0dd1760..5c8a98bb 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -26,7 +26,7 @@ use tokio::{ task::JoinHandle, }; -use crate::{globals::RoomMutexGuard, pdu::PduBuilder, services, user_is_local, PduEvent}; +use crate::{pdu::PduBuilder, rooms::state::RoomMutexGuard, services, user_is_local, PduEvent}; const COMMAND_QUEUE_LIMIT: usize = 512; @@ -248,7 +248,7 @@ async fn respond_to_room(content: RoomMessageEventContent, room_id: &RoomId, use "sender is not admin" ); - let state_lock = services().globals.roomid_mutex_state.lock(room_id).await; + let state_lock = services().rooms.state.mutex.lock(room_id).await; let response_pdu = PduBuilder { event_type: TimelineEventType::RoomMessage, content: to_raw_value(&content).expect("event is valid, we just created it"), diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index a5b70835..16830d87 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -12,19 +12,15 @@ use std::{ time::Instant, }; -use conduit::{ - error, trace, - utils::{MutexMap, MutexMapGuard}, - Config, Result, -}; +use conduit::{error, trace, Config, Result}; use data::Data; use ipaddress::IPAddress; use regex::RegexSet; use ruma::{ api::{client::discovery::discover_support::ContactRole, federation::discovery::VerifyKey}, serde::Base64, - DeviceId, OwnedEventId, OwnedRoomAliasId, OwnedRoomId, OwnedServerName, OwnedServerSigningKeyId, OwnedUserId, - RoomAliasId, RoomVersionId, ServerName, UserId, + DeviceId, OwnedEventId, OwnedRoomAliasId, OwnedServerName, OwnedServerSigningKeyId, OwnedUserId, RoomAliasId, + RoomVersionId, ServerName, UserId, }; use tokio::{sync::Mutex, task::JoinHandle}; use url::Url; @@ -45,18 +41,12 @@ pub struct Service { pub bad_event_ratelimiter: Arc>>, pub bad_signature_ratelimiter: Arc, RateLimitState>>>, pub bad_query_ratelimiter: Arc>>, - pub roomid_mutex_insert: RoomMutexMap, - pub roomid_mutex_state: RoomMutexMap, - pub roomid_mutex_federation: RoomMutexMap, - pub roomid_federationhandletime: RwLock>, pub updates_handle: Mutex>>, pub stateres_mutex: Arc>, pub server_user: OwnedUserId, pub admin_alias: OwnedRoomAliasId, } -pub type RoomMutexMap = MutexMap; -pub type RoomMutexGuard = MutexMapGuard; type RateLimitState = (Instant, u32); // Time if last failed try, number of failed tries impl crate::Service for Service { @@ -113,10 +103,6 @@ impl crate::Service for Service { bad_event_ratelimiter: Arc::new(RwLock::new(HashMap::new())), bad_signature_ratelimiter: Arc::new(RwLock::new(HashMap::new())), bad_query_ratelimiter: Arc::new(RwLock::new(HashMap::new())), - roomid_mutex_state: MutexMap::::new(), - roomid_mutex_insert: MutexMap::::new(), - roomid_mutex_federation: MutexMap::::new(), - roomid_federationhandletime: RwLock::new(HashMap::new()), updates_handle: Mutex::new(None), stateres_mutex: Arc::new(Mutex::new(())), admin_alias: RoomAliasId::parse(format!("#admins:{}", &config.server_name)) diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 53ce6f8d..bf553d5c 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -4,13 +4,14 @@ mod signing_keys; use std::{ collections::{hash_map, BTreeMap, HashMap, HashSet}, pin::Pin, - sync::Arc, + sync::{Arc, RwLock as StdRwLock}, time::Instant, }; use conduit::{ - debug, debug_error, debug_info, error, info, trace, utils::math::continue_exponential_backoff_secs, warn, Error, - Result, + debug, debug_error, debug_info, error, info, trace, + utils::{math::continue_exponential_backoff_secs, MutexMap}, + warn, Error, Result, }; use futures_util::Future; pub use parse_incoming_pdu::parse_incoming_pdu; @@ -28,14 +29,21 @@ use ruma::{ int, serde::Base64, state_res::{self, RoomVersion, StateMap}, - uint, CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, OwnedUserId, RoomId, RoomVersionId, ServerName, + uint, CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedRoomId, OwnedUserId, RoomId, + RoomVersionId, ServerName, }; use tokio::sync::RwLock; use super::state_compressor::CompressedStateEvent; use crate::{pdu, services, PduEvent}; -pub struct Service; +pub struct Service { + pub federation_handletime: StdRwLock, + pub mutex_federation: RoomMutexMap, +} + +type RoomMutexMap = MutexMap; +type HandleTimeMap = HashMap; // We use some AsyncRecursiveType hacks here so we can call async funtion // recursively. @@ -46,7 +54,12 @@ type AsyncRecursiveCanonicalJsonResult<'a> = AsyncRecursiveType<'a, Result<(Arc, BTreeMap)>>; impl crate::Service for Service { - fn build(_args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self {})) } + fn build(_args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + federation_handletime: HandleTimeMap::new().into(), + mutex_federation: RoomMutexMap::new(), + })) + } fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } @@ -200,9 +213,7 @@ impl Service { // Done with prev events, now handling the incoming event let start_time = Instant::now(); - services() - .globals - .roomid_federationhandletime + self.federation_handletime .write() .expect("locked") .insert(room_id.to_owned(), (event_id.to_owned(), start_time)); @@ -211,9 +222,7 @@ impl Service { .upgrade_outlier_to_timeline_pdu(incoming_pdu, val, &create_event, origin, room_id, pub_key_map) .await; - services() - .globals - .roomid_federationhandletime + self.federation_handletime .write() .expect("locked") .remove(&room_id.to_owned()); @@ -272,9 +281,7 @@ impl Service { } let start_time = Instant::now(); - services() - .globals - .roomid_federationhandletime + self.federation_handletime .write() .expect("locked") .insert(room_id.to_owned(), ((*prev_id).to_owned(), start_time)); @@ -282,9 +289,7 @@ impl Service { self.upgrade_outlier_to_timeline_pdu(pdu, json, create_event, origin, room_id, pub_key_map) .await?; - services() - .globals - .roomid_federationhandletime + self.federation_handletime .write() .expect("locked") .remove(&room_id.to_owned()); @@ -579,7 +584,7 @@ impl Service { // We start looking at current room state now, so lets lock the room trace!("Locking the room"); - let state_lock = services().globals.roomid_mutex_state.lock(room_id).await; + let state_lock = services().rooms.state.mutex.lock(room_id).await; // Now we calculate the set of extremities this room has after the incoming // event has been applied. We start with the previous extremities (aka leaves) diff --git a/src/service/rooms/state/data.rs b/src/service/rooms/state/data.rs index b62adf60..3c110afc 100644 --- a/src/service/rooms/state/data.rs +++ b/src/service/rooms/state/data.rs @@ -4,7 +4,7 @@ use conduit::{utils, Error, Result}; use database::{Database, Map}; use ruma::{EventId, OwnedEventId, RoomId}; -use crate::globals::RoomMutexGuard; +use super::RoomMutexGuard; pub(super) struct Data { shorteventid_shortstatehash: Arc, diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index b46a9d04..7d89ee33 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -5,7 +5,10 @@ use std::{ sync::Arc, }; -use conduit::{utils::calculate_hash, warn, Error, Result}; +use conduit::{ + utils::{calculate_hash, MutexMap, MutexMapGuard}, + warn, Error, Result, +}; use data::Data; use ruma::{ api::client::error::ErrorKind, @@ -15,20 +18,25 @@ use ruma::{ }, serde::Raw, state_res::{self, StateMap}, - EventId, OwnedEventId, RoomId, RoomVersionId, UserId, + EventId, OwnedEventId, OwnedRoomId, RoomId, RoomVersionId, UserId, }; use super::state_compressor::CompressedStateEvent; -use crate::{globals::RoomMutexGuard, services, PduEvent}; +use crate::{services, PduEvent}; pub struct Service { db: Data, + pub mutex: RoomMutexMap, } +type RoomMutexMap = MutexMap; +pub type RoomMutexGuard = MutexMapGuard; + impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { db: Data::new(args.db), + mutex: RoomMutexMap::new(), })) } diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 35719c15..389f12c3 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -33,7 +33,7 @@ use ruma::{ }; use serde_json::value::to_raw_value; -use crate::{globals::RoomMutexGuard, pdu::PduBuilder, services, PduEvent}; +use crate::{pdu::PduBuilder, rooms::state::RoomMutexGuard, services, PduEvent}; pub struct Service { db: Data, diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 9bfc2715..df2d46bd 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -6,7 +6,11 @@ use std::{ sync::Arc, }; -use conduit::{debug, error, info, utils, validated, warn, Error, Result}; +use conduit::{ + debug, error, info, utils, + utils::{MutexMap, MutexMapGuard}, + validated, warn, Error, Result, +}; use data::Data; use itertools::Itertools; use ruma::{ @@ -26,8 +30,8 @@ use ruma::{ push::{Action, Ruleset, Tweak}, serde::Base64, state_res::{self, Event, RoomVersion}, - uint, user_id, CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedEventId, OwnedServerName, RoomId, - RoomVersionId, ServerName, UserId, + uint, user_id, CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedEventId, OwnedRoomId, OwnedServerName, + RoomId, RoomVersionId, ServerName, UserId, }; use serde::Deserialize; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; @@ -36,7 +40,6 @@ use tokio::sync::RwLock; use crate::{ admin, appservice::NamespaceRegex, - globals::RoomMutexGuard, pdu::{EventHash, PduBuilder}, rooms::{event_handler::parse_incoming_pdu, state_compressor::CompressedStateEvent}, server_is_ours, services, PduCount, PduEvent, @@ -66,12 +69,17 @@ struct ExtractBody { pub struct Service { db: Data, + pub mutex_insert: RoomMutexMap, } +type RoomMutexMap = MutexMap; +pub type RoomMutexGuard = MutexMapGuard; + impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { db: Data::new(args.db), + mutex_insert: RoomMutexMap::new(), })) } @@ -269,11 +277,7 @@ impl Service { .state .set_forward_extremities(&pdu.room_id, leaves, state_lock)?; - let insert_lock = services() - .globals - .roomid_mutex_insert - .lock(&pdu.room_id) - .await; + let insert_lock = self.mutex_insert.lock(&pdu.room_id).await; let count1 = services().globals.next_count()?; // Mark as read first so the sending client doesn't get a notification even if @@ -1154,8 +1158,9 @@ impl Service { // Lock so we cannot backfill the same pdu twice at the same time let mutex_lock = services() - .globals - .roomid_mutex_federation + .rooms + .event_handler + .mutex_federation .lock(&room_id) .await; @@ -1187,7 +1192,7 @@ impl Service { .get_shortroomid(&room_id)? .expect("room exists"); - let insert_lock = services().globals.roomid_mutex_insert.lock(&room_id).await; + let insert_lock = self.mutex_insert.lock(&room_id).await; let max = u64::MAX; let count = services().globals.next_count()?; From 2a304c2b6c4f0b29afe1786e299f7cb1d1836f86 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 9 Jul 2024 21:22:44 +0000 Subject: [PATCH 091/158] add some usage stats output on the mutex maps Signed-off-by: Jason Volk --- src/service/rooms/event_handler/mod.rs | 15 +++++++++++++++ src/service/rooms/state/mod.rs | 8 ++++++++ src/service/rooms/timeline/mod.rs | 3 +++ 3 files changed, 26 insertions(+) diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index bf553d5c..9490a404 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -3,6 +3,7 @@ mod signing_keys; use std::{ collections::{hash_map, BTreeMap, HashMap, HashSet}, + fmt::Write, pin::Pin, sync::{Arc, RwLock as StdRwLock}, time::Instant, @@ -61,6 +62,20 @@ impl crate::Service for Service { })) } + fn memory_usage(&self, out: &mut dyn Write) -> Result<()> { + let mutex_federation = self.mutex_federation.len(); + writeln!(out, "federation_mutex: {mutex_federation}")?; + + let federation_handletime = self + .federation_handletime + .read() + .expect("locked for reading") + .len(); + writeln!(out, "federation_handletime: {federation_handletime}")?; + + Ok(()) + } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index 7d89ee33..a3a317a5 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -2,6 +2,7 @@ mod data; use std::{ collections::{HashMap, HashSet}, + fmt::Write, sync::Arc, }; @@ -40,6 +41,13 @@ impl crate::Service for Service { })) } + fn memory_usage(&self, out: &mut dyn Write) -> Result<()> { + let mutex = self.mutex.len(); + writeln!(out, "state_mutex: {mutex}")?; + + Ok(()) + } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index df2d46bd..e255e978 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -92,6 +92,9 @@ impl crate::Service for Service { .len(); writeln!(out, "lasttimelinecount_cache: {lasttimelinecount_cache}")?; + let mutex_insert = self.mutex_insert.len(); + writeln!(out, "insert_mutex: {mutex_insert}")?; + Ok(()) } From 16e76d45cb22d9c96d80a57a5ed5af810cc64946 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 9 Jul 2024 21:55:56 +0000 Subject: [PATCH 092/158] improve alloc stats interface; fix admin command formatting Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 4 ++-- src/admin/server/commands.rs | 13 ++++--------- src/core/alloc/default.rs | 8 ++++---- src/core/alloc/hardened.rs | 9 +++++---- src/core/alloc/je.rs | 10 +++++----- 5 files changed, 20 insertions(+), 24 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 62741f38..5fd0360d 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -632,12 +632,12 @@ pub(super) async fn resolve_true_destination( pub(super) fn memory_stats() -> RoomMessageEventContent { let html_body = conduit::alloc::memory_stats(); - if html_body.is_empty() { + if html_body.is_none() { return RoomMessageEventContent::text_plain("malloc stats are not supported on your compiled malloc."); } RoomMessageEventContent::text_html( "This command's output can only be viewed by clients that render HTML.".to_owned(), - html_body, + html_body.expect("string result"), ) } diff --git a/src/admin/server/commands.rs b/src/admin/server/commands.rs index 06007fb8..229f05e1 100644 --- a/src/admin/server/commands.rs +++ b/src/admin/server/commands.rs @@ -20,17 +20,12 @@ pub(super) async fn show_config(_body: Vec<&str>) -> Result) -> Result { - let response0 = services().memory_usage().await?; - let response1 = services().db.db.memory_usage()?; - let response2 = conduit::alloc::memory_usage(); + let services_usage = services().memory_usage().await?; + let database_usage = services().db.db.memory_usage()?; + let allocator_usage = conduit::alloc::memory_usage().map_or(String::new(), |s| format!("\nAllocator:\n{s}")); Ok(RoomMessageEventContent::text_plain(format!( - "Services:\n{response0}\nDatabase:\n{response1}\n{}", - if !response2.is_empty() { - format!("Allocator:\n {response2}") - } else { - String::new() - } + "Services:\n{services_usage}\nDatabase:\n{database_usage}{allocator_usage}", ))) } diff --git a/src/core/alloc/default.rs b/src/core/alloc/default.rs index 4e2f8d7e..83bfca7d 100644 --- a/src/core/alloc/default.rs +++ b/src/core/alloc/default.rs @@ -1,9 +1,9 @@ //! Default allocator with no special features -/// Always returns the empty string +/// Always returns None #[must_use] -pub fn memory_stats() -> String { String::default() } +pub fn memory_stats() -> Option { None } -/// Always returns the empty string +/// Always returns None #[must_use] -pub fn memory_usage() -> String { String::default() } +pub fn memory_usage() -> Option { None } diff --git a/src/core/alloc/hardened.rs b/src/core/alloc/hardened.rs index 6727407f..335a3307 100644 --- a/src/core/alloc/hardened.rs +++ b/src/core/alloc/hardened.rs @@ -4,9 +4,10 @@ static HMALLOC: hardened_malloc_rs::HardenedMalloc = hardened_malloc_rs::HardenedMalloc; #[must_use] -pub fn memory_usage() -> String { - String::default() //TODO: get usage -} +//TODO: get usage +pub fn memory_usage() -> Option { None } #[must_use] -pub fn memory_stats() -> String { "Extended statistics are not available from hardened_malloc.".to_owned() } +pub fn memory_stats() -> Option { + Some("Extended statistics are not available from hardened_malloc.".to_owned()) +} diff --git a/src/core/alloc/je.rs b/src/core/alloc/je.rs index 5e3c361f..08bfc49a 100644 --- a/src/core/alloc/je.rs +++ b/src/core/alloc/je.rs @@ -10,7 +10,7 @@ use tikv_jemallocator as jemalloc; static JEMALLOC: jemalloc::Jemalloc = jemalloc::Jemalloc; #[must_use] -pub fn memory_usage() -> String { +pub fn memory_usage() -> Option { use mallctl::stats; let mibs = |input: Result| { @@ -27,14 +27,14 @@ pub fn memory_usage() -> String { let metadata = mibs(stats::metadata::read()); let resident = mibs(stats::resident::read()); let retained = mibs(stats::retained::read()); - format!( + Some(format!( "allocated: {allocated:.2} MiB\nactive: {active:.2} MiB\nmapped: {mapped:.2} MiB\nmetadata: {metadata:.2} \ MiB\nresident: {resident:.2} MiB\nretained: {retained:.2} MiB\n" - ) + )) } #[must_use] -pub fn memory_stats() -> String { +pub fn memory_stats() -> Option { const MAX_LENGTH: usize = 65536 - 4096; let opts_s = "d"; @@ -51,7 +51,7 @@ pub fn memory_stats() -> String { unsafe { ffi::malloc_stats_print(Some(malloc_stats_cb), opaque, opts_p) }; str.truncate(MAX_LENGTH); - format!("
{str}
") + Some(format!("
{str}
")) } extern "C" fn malloc_stats_cb(opaque: *mut c_void, msg: *const c_char) { From f10f5319db5b0189901008487f0b9b5b4b375e58 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 10 Jul 2024 06:34:03 +0000 Subject: [PATCH 093/158] elaborate error log functor stack Signed-off-by: Jason Volk --- src/core/error.rs | 70 ++++++++++++++++++++++++++++++------ src/service/admin/console.rs | 5 ++- 2 files changed, 63 insertions(+), 12 deletions(-) diff --git a/src/core/error.rs b/src/core/error.rs index 1959081a..7069d1e6 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -117,25 +117,73 @@ impl Error { } #[inline] -pub fn log(e: &Error) { - error!(?e); +pub fn else_log(error: E) -> Result +where + T: Default, + Error: From, +{ + Ok(default_log(error)) } #[inline] -pub fn debug_log(e: &Error) { - debug_error!(?e); +pub fn else_debug_log(error: E) -> Result +where + T: Default, + Error: From, +{ + Ok(default_debug_log(error)) } #[inline] -pub fn into_log(e: Error) { - error!(?e); - drop(e); +pub fn default_log(error: E) -> T +where + T: Default, + Error: From, +{ + let error = Error::from(error); + inspect_log(&error); + T::default() } #[inline] -pub fn into_debug_log(e: Error) { - debug_error!(?e); - drop(e); +pub fn default_debug_log(error: E) -> T +where + T: Default, + Error: From, +{ + let error = Error::from(error); + inspect_debug_log(&error); + T::default() +} + +#[inline] +pub fn map_log(error: E) -> Error +where + Error: From, +{ + let error = Error::from(error); + inspect_log(&error); + error +} + +#[inline] +pub fn map_debug_log(error: E) -> Error +where + Error: From, +{ + let error = Error::from(error); + inspect_debug_log(&error); + error +} + +#[inline] +pub fn inspect_log(error: &E) { + error!("{error}"); +} + +#[inline] +pub fn inspect_debug_log(error: &E) { + debug_error!("{error:?}"); } impl fmt::Debug for Error { @@ -151,7 +199,7 @@ impl axum::response::IntoResponse for Error { let response: UiaaResponse = self.into(); response .try_into_http_response::() - .inspect_err(|e| error!(?e)) + .inspect_err(|e| error!("error response error: {e}")) .map_or_else( |_| StatusCode::INTERNAL_SERVER_ERROR.into_response(), |r| r.map(BytesMut::freeze).map(Full::new).into_response(), diff --git a/src/service/admin/console.rs b/src/service/admin/console.rs index 2f66b1d5..c590b928 100644 --- a/src/service/admin/console.rs +++ b/src/service/admin/console.rs @@ -95,7 +95,10 @@ impl Console { ReadlineEvent::Line(string) => self.clone().handle(string).await, ReadlineEvent::Interrupted => continue, ReadlineEvent::Eof => break, - ReadlineEvent::Quit => services().server.shutdown().unwrap_or_else(error::into_log), + ReadlineEvent::Quit => services() + .server + .shutdown() + .unwrap_or_else(error::default_log), }, Err(error) => match error { ReadlineError::Closed => break, From 0627b46f4018e5a589bc106ebae24cb60cc91ae2 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 10 Jul 2024 06:35:11 +0000 Subject: [PATCH 094/158] add panic suite to Error Signed-off-by: Jason Volk --- src/api/server/send.rs | 2 +- src/core/debug.rs | 7 +++-- src/core/error.rs | 71 ++++++++++++++++++++++++++++++++---------- 3 files changed, 60 insertions(+), 20 deletions(-) diff --git a/src/api/server/send.rs b/src/api/server/send.rs index 08caf1b4..122f564f 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -84,7 +84,7 @@ pub(crate) async fn send_transaction_message_route( Ok(send_transaction_message::v1::Response { pdus: resolved_map .into_iter() - .map(|(e, r)| (e, r.map_err(|e| e.sanitized_error()))) + .map(|(e, r)| (e, r.map_err(|e| e.sanitized_string()))) .collect(), }) } diff --git a/src/core/debug.rs b/src/core/debug.rs index 1f855e52..14d0be87 100644 --- a/src/core/debug.rs +++ b/src/core/debug.rs @@ -1,6 +1,4 @@ -#![allow(dead_code)] // this is a developer's toolbox - -use std::panic; +use std::{any::Any, panic}; /// Export all of the ancillary tools from here as well. pub use crate::utils::debug::*; @@ -79,3 +77,6 @@ pub fn trap() { std::arch::asm!("int3"); } } + +#[must_use] +pub fn panic_str(p: &Box) -> &'static str { p.downcast_ref::<&str>().copied().unwrap_or_default() } diff --git a/src/core/error.rs b/src/core/error.rs index 7069d1e6..26bac6e0 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -1,4 +1,9 @@ -use std::{convert::Infallible, fmt}; +use std::{ + any::Any, + convert::Infallible, + fmt, + panic::{RefUnwindSafe, UnwindSafe}, +}; use bytes::BytesMut; use http::StatusCode; @@ -8,10 +13,15 @@ use ruma::{ OwnedServerName, }; -use crate::{debug_error, error}; +use crate::{debug::panic_str, debug_error, error}; #[derive(thiserror::Error)] pub enum Error { + #[error("PANIC!")] + PanicAny(Box), + #[error("PANIC! {0}")] + Panic(&'static str, Box), + // std #[error("{0}")] Fmt(#[from] fmt::Error), @@ -31,6 +41,8 @@ pub enum Error { ParseFloatError(#[from] std::num::ParseFloatError), // third-party + #[error("Join error: {0}")] + JoinError(#[from] tokio::task::JoinError), #[error("Regex error: {0}")] Regex(#[from] regex::Error), #[error("Tracing filter error: {0}")] @@ -94,6 +106,29 @@ impl Error { Self::BadConfig(message.to_owned()) } + #[must_use] + pub fn from_panic(e: Box) -> Self { Self::Panic(panic_str(&e), e) } + + pub fn into_panic(self) -> Box { + match self { + Self::Panic(_, e) | Self::PanicAny(e) => e, + Self::JoinError(e) => e.into_panic(), + _ => std::panic::panic_any(self), + } + } + + /// Sanitizes public-facing errors that can leak sensitive information. + pub fn sanitized_string(&self) -> String { + match self { + Self::Database(..) => String::from("Database error occurred."), + Self::Io(..) => String::from("I/O error occurred."), + _ => self.to_string(), + } + } + + /// Get the panic message string. + pub fn panic_str(self) -> Option<&'static str> { self.is_panic().then_some(panic_str(&self.into_panic())) } + /// Returns the Matrix error code / error kind #[inline] pub fn error_code(&self) -> ruma::api::client::error::ErrorKind { @@ -106,16 +141,28 @@ impl Error { } } - /// Sanitizes public-facing errors that can leak sensitive information. - pub fn sanitized_error(&self) -> String { - match self { - Self::Database(..) => String::from("Database error occurred."), - Self::Io(..) => String::from("I/O error occurred."), - _ => self.to_string(), + /// Check if the Error is trafficking a panic object. + #[inline] + pub fn is_panic(&self) -> bool { + match &self { + Self::Panic(..) | Self::PanicAny(..) => true, + Self::JoinError(e) => e.is_panic(), + _ => false, } } } +impl UnwindSafe for Error {} +impl RefUnwindSafe for Error {} + +impl From for Error { + fn from(i: Infallible) -> Self { match i {} } +} + +impl fmt::Debug for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{self}") } +} + #[inline] pub fn else_log(error: E) -> Result where @@ -186,14 +233,6 @@ pub fn inspect_debug_log(error: &E) { debug_error!("{error:?}"); } -impl fmt::Debug for Error { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{self}") } -} - -impl From for Error { - fn from(i: Infallible) -> Self { match i {} } -} - impl axum::response::IntoResponse for Error { fn into_response(self) -> axum::response::Response { let response: UiaaResponse = self.into(); From 61f2a3c68b433be1a8acb96f79f31cc746a9b311 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 10 Jul 2024 10:43:41 +0000 Subject: [PATCH 095/158] catch panic from admin commands Signed-off-by: Jason Volk --- Cargo.lock | 1 + src/admin/Cargo.toml | 1 + src/admin/handler.rs | 46 ++++++++++++++++++++++++++++++++------------ 3 files changed, 36 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1846c82d..02da0b6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -613,6 +613,7 @@ dependencies = [ "conduit_api", "conduit_core", "conduit_service", + "futures-util", "log", "ruma", "serde_json", diff --git a/src/admin/Cargo.toml b/src/admin/Cargo.toml index f84fbb92..7a092237 100644 --- a/src/admin/Cargo.toml +++ b/src/admin/Cargo.toml @@ -30,6 +30,7 @@ clap.workspace = true conduit-api.workspace = true conduit-core.workspace = true conduit-service.workspace = true +futures-util.workspace = true log.workspace = true ruma.workspace = true serde_json.workspace = true diff --git a/src/admin/handler.rs b/src/admin/handler.rs index 22ec81c5..2d1f96b5 100644 --- a/src/admin/handler.rs +++ b/src/admin/handler.rs @@ -1,10 +1,14 @@ -use std::time::Instant; +use std::{panic::AssertUnwindSafe, time::Instant}; use clap::{CommandFactory, Parser}; -use conduit::trace; -use ruma::events::{ - relation::InReplyTo, - room::message::{Relation::Reply, RoomMessageEventContent}, +use conduit::{error, trace, Error}; +use futures_util::future::FutureExt; +use ruma::{ + events::{ + relation::InReplyTo, + room::message::{Relation::Reply, RoomMessageEventContent}, + }, + OwnedEventId, }; extern crate conduit_service as service; @@ -69,21 +73,39 @@ pub(crate) fn complete(line: &str) -> String { complete_admin_command(AdminComma #[tracing::instrument(skip_all, name = "admin")] async fn handle_command(command: Command) -> CommandResult { - let Some(mut content) = process_admin_message(command.command).await else { - return Ok(None); - }; + AssertUnwindSafe(process_command(&command)) + .catch_unwind() + .await + .map_err(Error::from_panic) + .or_else(|error| handle_panic(&error, command)) +} - content.relates_to = command.reply_id.map(|event_id| Reply { +async fn process_command(command: &Command) -> CommandOutput { + process_admin_message(&command.command) + .await + .and_then(|content| reply(content, command.reply_id.clone())) +} + +fn handle_panic(error: &Error, command: Command) -> CommandResult { + let link = "Please submit a [bug report](https://github.com/girlbossceo/conduwuit/issues/new). 🥺"; + let msg = format!("Panic occurred while processing command:\n```\n{error:#?}\n```\n{link}"); + let content = RoomMessageEventContent::notice_markdown(msg); + error!("Panic while processing command: {error:?}"); + Ok(reply(content, command.reply_id)) +} + +fn reply(mut content: RoomMessageEventContent, reply_id: Option) -> Option { + content.relates_to = reply_id.map(|event_id| Reply { in_reply_to: InReplyTo { event_id, }, }); - Ok(Some(content)) + Some(content) } // Parse and process a message from the admin room -async fn process_admin_message(msg: String) -> CommandOutput { +async fn process_admin_message(msg: &str) -> CommandOutput { let mut lines = msg.lines().filter(|l| !l.trim().is_empty()); let command = lines.next().expect("each string has at least one line"); let body = lines.collect::>(); @@ -103,7 +125,7 @@ async fn process_admin_message(msg: String) -> CommandOutput { match result { Ok(reply) => Some(reply), Err(error) => Some(RoomMessageEventContent::notice_markdown(format!( - "Encountered an error while handling the command:\n```\n{error}\n```" + "Encountered an error while handling the command:\n```\n{error:#?}\n```" ))), } } From e73aa2aa2146ebd99f56dea9ad4c5f5d51bf9fa6 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 11 Jul 2024 02:29:06 +0000 Subject: [PATCH 096/158] log propagated errors in admin service; minor cleanup Signed-off-by: Jason Volk --- src/service/admin/mod.rs | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index 5c8a98bb..e9729d2d 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -9,7 +9,7 @@ use std::{ }; use async_trait::async_trait; -use conduit::{error, Error, Result}; +use conduit::{debug, error, Error, Result}; pub use create::create_admin_room; pub use grant::make_user_admin; use loole::{Receiver, Sender}; @@ -28,14 +28,6 @@ use tokio::{ use crate::{pdu::PduBuilder, rooms::state::RoomMutexGuard, services, user_is_local, PduEvent}; -const COMMAND_QUEUE_LIMIT: usize = 512; - -pub type CommandOutput = Option; -pub type CommandResult = Result; -pub type HandlerResult = Pin + Send>>; -pub type Handler = fn(Command) -> HandlerResult; -pub type Completer = fn(&str) -> String; - pub struct Service { sender: Sender, receiver: Mutex>, @@ -52,6 +44,14 @@ pub struct Command { pub reply_id: Option, } +pub type Completer = fn(&str) -> String; +pub type Handler = fn(Command) -> HandlerResult; +pub type HandlerResult = Pin + Send>>; +pub type CommandResult = Result; +pub type CommandOutput = Option; + +const COMMAND_QUEUE_LIMIT: usize = 512; + #[async_trait] impl crate::Service for Service { fn build(_args: crate::Args<'_>) -> Result> { @@ -172,8 +172,10 @@ impl Service { } async fn handle_command(&self, command: Command) { - if let Ok(Some(output)) = self.process_command(command).await { - handle_response(output).await; + match self.process_command(command).await { + Ok(Some(output)) => handle_response(output).await, + Ok(None) => debug!("Command successful with no response"), + Err(e) => error!("Command processing error: {e}"), } } @@ -263,14 +265,14 @@ async fn respond_to_room(content: RoomMessageEventContent, room_id: &RoomId, use .build_and_append_pdu(response_pdu, user_id, room_id, &state_lock) .await { - if let Err(e) = handle_response_error(&e, room_id, user_id, &state_lock).await { + if let Err(e) = handle_response_error(e, room_id, user_id, &state_lock).await { error!("{e}"); } } } async fn handle_response_error( - e: &Error, room_id: &RoomId, user_id: &UserId, state_lock: &RoomMutexGuard, + e: Error, room_id: &RoomId, user_id: &UserId, state_lock: &RoomMutexGuard, ) -> Result<()> { error!("Failed to build and append admin room response PDU: \"{e}\""); let error_room_message = RoomMessageEventContent::text_plain(format!( From db3c718ddc224a15b0dcbc208413c8a8a8242766 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 10 Jul 2024 20:27:17 +0000 Subject: [PATCH 097/158] add config for sentry stacktraces Signed-off-by: Jason Volk --- conduwuit-example.toml | 3 +++ src/core/config/mod.rs | 3 +++ src/main/sentry.rs | 1 + 3 files changed, 7 insertions(+) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 7e5c9710..cb6dc8e6 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -57,6 +57,9 @@ # Defaults to 0.15 #sentry_traces_sample_rate = 0.15 +# Whether to attach a stacktrace to Sentry reports. +#sentry_attach_stacktrace = false + ### Database configuration diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index f9a5ec4c..8b864c84 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -356,6 +356,8 @@ pub struct Config { pub sentry_send_server_name: bool, #[serde(default = "default_sentry_traces_sample_rate")] pub sentry_traces_sample_rate: f32, + #[serde(default)] + pub sentry_attach_stacktrace: bool, #[serde(default)] pub tokio_console: bool, @@ -819,6 +821,7 @@ impl fmt::Display for Config { ("Sentry.io send server_name in logs", &self.sentry_send_server_name.to_string()), #[cfg(feature = "sentry_telemetry")] ("Sentry.io tracing sample rate", &self.sentry_traces_sample_rate.to_string()), + ("Sentry.io attach stacktrace", &self.sentry_attach_stacktrace.to_string()), ( "Well-known server name", self.well_known diff --git a/src/main/sentry.rs b/src/main/sentry.rs index 6ed4bb8a..59f88a0c 100644 --- a/src/main/sentry.rs +++ b/src/main/sentry.rs @@ -28,6 +28,7 @@ fn options(config: &Config) -> ClientOptions { debug: cfg!(debug_assertions), release: sentry::release_name!(), user_agent: conduit::version::user_agent().into(), + attach_stacktrace: config.sentry_attach_stacktrace, before_send: Some(Arc::new(before_send)), before_breadcrumb: Some(Arc::new(before_breadcrumb)), ..Default::default() From 0023b09f5b644cf51597cf010f643e37af3d98bb Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 10 Jul 2024 22:28:56 +0000 Subject: [PATCH 098/158] make tracing reload handles into a named map Signed-off-by: Jason Volk --- src/core/error.rs | 2 ++ src/core/log/reload.rs | 50 ++++++++++++++++++++++++++---------------- src/main/tracing.rs | 13 ++++++----- 3 files changed, 40 insertions(+), 25 deletions(-) diff --git a/src/core/error.rs b/src/core/error.rs index 26bac6e0..8567f303 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -47,6 +47,8 @@ pub enum Error { Regex(#[from] regex::Error), #[error("Tracing filter error: {0}")] TracingFilter(#[from] tracing_subscriber::filter::ParseError), + #[error("Tracing reload error: {0}")] + TracingReload(#[from] tracing_subscriber::reload::Error), #[error("Image error: {0}")] Image(#[from] image::error::ImageError), #[error("Request error: {0}")] diff --git a/src/core/log/reload.rs b/src/core/log/reload.rs index 7646254e..cfed785d 100644 --- a/src/core/log/reload.rs +++ b/src/core/log/reload.rs @@ -1,7 +1,12 @@ -use std::sync::Arc; +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; use tracing_subscriber::{reload, EnvFilter}; +use crate::{error, Result}; + /// We need to store a reload::Handle value, but can't name it's type explicitly /// because the S type parameter depends on the subscriber's previous layers. In /// our case, this includes unnameable 'impl Trait' types. @@ -24,32 +29,39 @@ impl ReloadHandle for reload::Handle { fn reload(&self, new_value: L) -> Result<(), reload::Error> { Self::reload(self, new_value) } } -struct LogLevelReloadHandlesInner { - handles: Vec + Send + Sync>>, -} - -/// Wrapper to allow reloading the filter on several several -/// [`tracing_subscriber::reload::Handle`]s at once, with the same value. #[derive(Clone)] pub struct LogLevelReloadHandles { - inner: Arc, + handles: Arc>, } +type HandleMap = HashMap; +type Handle = Box + Send + Sync>; + impl LogLevelReloadHandles { - #[must_use] - pub fn new(handles: Vec + Send + Sync>>) -> Self { - Self { - inner: Arc::new(LogLevelReloadHandlesInner { - handles, - }), - } + pub fn add(&self, name: &str, handle: Handle) { + self.handles + .lock() + .expect("locked") + .insert(name.into(), handle); } - pub fn reload(&self, new_value: &EnvFilter) -> Result<(), reload::Error> { - for handle in &self.inner.handles { - handle.reload(new_value.clone())?; - } + pub fn reload(&self, new_value: &EnvFilter) -> Result<()> { + self.handles + .lock() + .expect("locked") + .values() + .for_each(|handle| { + _ = handle.reload(new_value.clone()).or_else(error::else_log); + }); Ok(()) } } + +impl Default for LogLevelReloadHandles { + fn default() -> Self { + Self { + handles: Arc::new(HandleMap::new().into()), + } + } +} diff --git a/src/main/tracing.rs b/src/main/tracing.rs index bbfe4dc4..4c8fde5a 100644 --- a/src/main/tracing.rs +++ b/src/main/tracing.rs @@ -4,7 +4,7 @@ use conduit::{ config, config::Config, debug_warn, - log::{capture, LogLevelReloadHandles, ReloadHandle}, + log::{capture, LogLevelReloadHandles}, }; use tracing_subscriber::{layer::SubscriberExt, reload, EnvFilter, Layer, Registry}; @@ -15,6 +15,8 @@ pub(crate) type TracingFlameGuard = (); #[allow(clippy::redundant_clone)] pub(crate) fn init(config: &Config) -> (LogLevelReloadHandles, TracingFlameGuard, Arc) { + let reload_handles = LogLevelReloadHandles::default(); + let fmt_layer = tracing_subscriber::fmt::Layer::new(); let filter_layer = match EnvFilter::try_new(&config.log) { Ok(s) => s, @@ -24,9 +26,8 @@ pub(crate) fn init(config: &Config) -> (LogLevelReloadHandles, TracingFlameGuard }, }; - let mut reload_handles = Vec:: + Send + Sync>>::new(); let (fmt_reload_filter, fmt_reload_handle) = reload::Layer::new(filter_layer.clone()); - reload_handles.push(Box::new(fmt_reload_handle)); + reload_handles.add("format", Box::new(fmt_reload_handle)); let subscriber = Registry::default().with(fmt_layer.with_filter(fmt_reload_filter)); @@ -38,7 +39,7 @@ pub(crate) fn init(config: &Config) -> (LogLevelReloadHandles, TracingFlameGuard let subscriber = { let sentry_layer = sentry_tracing::layer(); let (sentry_reload_filter, sentry_reload_handle) = reload::Layer::new(filter_layer.clone()); - reload_handles.push(Box::new(sentry_reload_handle)); + reload_handles.add("sentry", Box::new(sentry_reload_handle)); subscriber.with(sentry_layer.with_filter(sentry_reload_filter)) }; @@ -73,7 +74,7 @@ pub(crate) fn init(config: &Config) -> (LogLevelReloadHandles, TracingFlameGuard let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); let (jaeger_reload_filter, jaeger_reload_handle) = reload::Layer::new(filter_layer.clone()); - reload_handles.push(Box::new(jaeger_reload_handle)); + reload_handles.add("jaeger", Box::new(jaeger_reload_handle)); Some(telemetry.with_filter(jaeger_reload_filter)) } else { None @@ -87,7 +88,7 @@ pub(crate) fn init(config: &Config) -> (LogLevelReloadHandles, TracingFlameGuard #[cfg_attr(not(feature = "perf_measurements"), allow(clippy::let_unit_value))] let flame_guard = (); - let ret = (LogLevelReloadHandles::new(reload_handles), flame_guard, cap_state); + let ret = (reload_handles, flame_guard, cap_state); // Enable the tokio console. This is slightly kludgy because we're judggling // compile-time and runtime conditions to elide it, each of those changing the From 160f48043e55cc579d6bb77eb12924a11e134e65 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 11 Jul 2024 00:42:49 +0000 Subject: [PATCH 099/158] add selective log handle reload Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 16 ++++++++++++++-- src/core/log/reload.rs | 7 ++++--- src/core/log/suppress.rs | 4 ++-- src/main/tracing.rs | 2 +- 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 5fd0360d..a5087467 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -314,6 +314,8 @@ pub(super) async fn force_device_list_updates(_body: Vec<&str>) -> Result, filter: Option, reset: bool, ) -> Result { + let handles = &["console"]; + if reset { let old_filter_layer = match EnvFilter::try_new(&services().globals.config.log) { Ok(s) => s, @@ -324,7 +326,12 @@ pub(super) async fn change_log_level( }, }; - match services().server.log.reload.reload(&old_filter_layer) { + match services() + .server + .log + .reload + .reload(&old_filter_layer, Some(handles)) + { Ok(()) => { return Ok(RoomMessageEventContent::text_plain(format!( "Successfully changed log level back to config value {}", @@ -349,7 +356,12 @@ pub(super) async fn change_log_level( }, }; - match services().server.log.reload.reload(&new_filter_layer) { + match services() + .server + .log + .reload + .reload(&new_filter_layer, Some(handles)) + { Ok(()) => { return Ok(RoomMessageEventContent::text_plain("Successfully changed log level")); }, diff --git a/src/core/log/reload.rs b/src/core/log/reload.rs index cfed785d..0b72d787 100644 --- a/src/core/log/reload.rs +++ b/src/core/log/reload.rs @@ -45,12 +45,13 @@ impl LogLevelReloadHandles { .insert(name.into(), handle); } - pub fn reload(&self, new_value: &EnvFilter) -> Result<()> { + pub fn reload(&self, new_value: &EnvFilter, names: Option<&[&str]>) -> Result<()> { self.handles .lock() .expect("locked") - .values() - .for_each(|handle| { + .iter() + .filter(|(name, _)| names.map_or(false, |names| names.contains(&name.as_str()))) + .for_each(|(_, handle)| { _ = handle.reload(new_value.clone()).or_else(error::else_log); }); diff --git a/src/core/log/suppress.rs b/src/core/log/suppress.rs index 6e883086..ca2d5485 100644 --- a/src/core/log/suppress.rs +++ b/src/core/log/suppress.rs @@ -18,7 +18,7 @@ impl Suppress { server .log .reload - .reload(suppress) + .reload(suppress, Some(&["console"])) .expect("log filter reloaded"); Self { server: server.clone(), @@ -32,7 +32,7 @@ impl Drop for Suppress { self.server .log .reload - .reload(&self.restore) + .reload(&self.restore, Some(&["console"])) .expect("log filter reloaded"); } } diff --git a/src/main/tracing.rs b/src/main/tracing.rs index 4c8fde5a..78c35c66 100644 --- a/src/main/tracing.rs +++ b/src/main/tracing.rs @@ -27,7 +27,7 @@ pub(crate) fn init(config: &Config) -> (LogLevelReloadHandles, TracingFlameGuard }; let (fmt_reload_filter, fmt_reload_handle) = reload::Layer::new(filter_layer.clone()); - reload_handles.add("format", Box::new(fmt_reload_handle)); + reload_handles.add("console", Box::new(fmt_reload_handle)); let subscriber = Registry::default().with(fmt_layer.with_filter(fmt_reload_filter)); From bc58e5002d95ae86831fd04f8e059798188b7bb8 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 11 Jul 2024 01:02:05 +0000 Subject: [PATCH 100/158] add interface for current log filter; fix console suppression Signed-off-by: Jason Volk --- src/core/log/reload.rs | 15 ++++++++++++++- src/core/log/suppress.rs | 13 +++++++++---- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/core/log/reload.rs b/src/core/log/reload.rs index 0b72d787..6d651065 100644 --- a/src/core/log/reload.rs +++ b/src/core/log/reload.rs @@ -22,10 +22,14 @@ use crate::{error, Result}; /// /// [1]: pub trait ReloadHandle { + fn current(&self) -> Option; + fn reload(&self, new_value: L) -> Result<(), reload::Error>; } -impl ReloadHandle for reload::Handle { +impl ReloadHandle for reload::Handle { + fn current(&self) -> Option { Self::clone_current(self) } + fn reload(&self, new_value: L) -> Result<(), reload::Error> { Self::reload(self, new_value) } } @@ -57,6 +61,15 @@ impl LogLevelReloadHandles { Ok(()) } + + #[must_use] + pub fn current(&self, name: &str) -> Option { + self.handles + .lock() + .expect("locked") + .get(name) + .map(|handle| handle.current())? + } } impl Default for LogLevelReloadHandles { diff --git a/src/core/log/suppress.rs b/src/core/log/suppress.rs index ca2d5485..b13ee99e 100644 --- a/src/core/log/suppress.rs +++ b/src/core/log/suppress.rs @@ -10,16 +10,21 @@ pub struct Suppress { impl Suppress { pub fn new(server: &Arc) -> Self { + let handle = "console"; let config = &server.config.log; - Self::from_filters(server, EnvFilter::try_new(config).unwrap_or_default(), &EnvFilter::default()) - } + let suppress = EnvFilter::default(); + let restore = server + .log + .reload + .current(handle) + .unwrap_or_else(|| EnvFilter::try_new(config).unwrap_or_default()); - fn from_filters(server: &Arc, restore: EnvFilter, suppress: &EnvFilter) -> Self { server .log .reload - .reload(suppress, Some(&["console"])) + .reload(&suppress, Some(&[handle])) .expect("log filter reloaded"); + Self { server: server.clone(), restore, From d88ab371200c4bebf21a32c3afb43897af985df4 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 11 Jul 2024 02:04:21 +0000 Subject: [PATCH 101/158] add configuration for sentry to send panics and errors Signed-off-by: Jason Volk --- conduwuit-example.toml | 7 +++++ src/core/config/mod.rs | 6 +++++ src/main/sentry.rs | 61 ++++++++++++++++++++++++++++++++++++------ 3 files changed, 66 insertions(+), 8 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index cb6dc8e6..ea095bfa 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -60,6 +60,13 @@ # Whether to attach a stacktrace to Sentry reports. #sentry_attach_stacktrace = false +# Send panics to sentry. This is true by default, but sentry has to be enabled. +#sentry_send_panic = true + +# Send errors to sentry. This is true by default, but sentry has to be enabled. This option is +# only effective in release-mode; forced to false in debug-mode. +#sentry_send_error = true + ### Database configuration diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 8b864c84..1997a39e 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -358,6 +358,10 @@ pub struct Config { pub sentry_traces_sample_rate: f32, #[serde(default)] pub sentry_attach_stacktrace: bool, + #[serde(default = "true_fn")] + pub sentry_send_panic: bool, + #[serde(default = "true_fn")] + pub sentry_send_error: bool, #[serde(default)] pub tokio_console: bool, @@ -822,6 +826,8 @@ impl fmt::Display for Config { #[cfg(feature = "sentry_telemetry")] ("Sentry.io tracing sample rate", &self.sentry_traces_sample_rate.to_string()), ("Sentry.io attach stacktrace", &self.sentry_attach_stacktrace.to_string()), + ("Sentry.io send panics", &self.sentry_send_panic.to_string()), + ("Sentry.io send errors", &self.sentry_send_error.to_string()), ( "Well-known server name", self.well_known diff --git a/src/main/sentry.rs b/src/main/sentry.rs index 59f88a0c..04ad8654 100644 --- a/src/main/sentry.rs +++ b/src/main/sentry.rs @@ -1,18 +1,34 @@ #![cfg(feature = "sentry_telemetry")] -use std::{str::FromStr, sync::Arc}; - -use conduit::{config::Config, trace}; -use sentry::{ - types::{protocol::v7::Event, Dsn}, - Breadcrumb, ClientOptions, +use std::{ + str::FromStr, + sync::{Arc, OnceLock}, }; +use conduit::{config::Config, debug, trace}; +use sentry::{ + types::{ + protocol::v7::{Context, Event}, + Dsn, + }, + Breadcrumb, ClientOptions, Level, +}; + +static SEND_PANIC: OnceLock = OnceLock::new(); +static SEND_ERROR: OnceLock = OnceLock::new(); + pub(crate) fn init(config: &Config) -> Option { config.sentry.then(|| sentry::init(options(config))) } fn options(config: &Config) -> ClientOptions { + SEND_PANIC + .set(config.sentry_send_panic) + .expect("SEND_PANIC was not previously set"); + SEND_ERROR + .set(config.sentry_send_error) + .expect("SEND_ERROR was not previously set"); + let dsn = config .sentry_endpoint .as_ref() @@ -36,11 +52,40 @@ fn options(config: &Config) -> ClientOptions { } fn before_send(event: Event<'static>) -> Option> { - trace!("Sending sentry event: {event:?}"); + if event.exception.iter().any(|e| e.ty == "panic") && !SEND_PANIC.get().unwrap_or(&true) { + return None; + } + + if event.level == Level::Error { + if !SEND_ERROR.get().unwrap_or(&true) { + return None; + } + + if cfg!(debug_assertions) { + return None; + } + + //NOTE: we can enable this to specify error!(sentry = true, ...) + if let Some(Context::Other(context)) = event.contexts.get("Rust Tracing Fields") { + if !context.contains_key("sentry") { + //return None; + } + } + } + + if event.level == Level::Fatal { + trace!("{event:#?}"); + } + + debug!("Sending sentry event: {event:?}"); Some(event) } fn before_breadcrumb(crumb: Breadcrumb) -> Option { - trace!("Adding sentry breadcrumb: {crumb:?}"); + if crumb.ty == "log" && crumb.level == Level::Debug { + return None; + } + + trace!("Sentry breadcrumb: {crumb:?}"); Some(crumb) } From 57969f9480f5ad2fc4d392043be4003071a52890 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 11 Jul 2024 02:22:37 +0000 Subject: [PATCH 102/158] fix large stack array exceeding limit Signed-off-by: Jason Volk --- src/core/config/mod.rs | 710 ++++++++++++++++++++--------------------- 1 file changed, 353 insertions(+), 357 deletions(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 1997a39e..df13ad1f 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1,6 +1,6 @@ use std::{ collections::BTreeMap, - fmt::{self, Write as _}, + fmt, net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, path::PathBuf, }; @@ -522,364 +522,360 @@ impl Config { impl fmt::Display for Config { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - // Prepare a list of config values to show - let lines = [ - ("Server name", self.server_name.host()), - ("Database backend", &self.database_backend), - ("Database path", &self.database_path.to_string_lossy()), - ( - "Database backup path", - self.database_backup_path - .as_ref() - .map_or("", |path| path.to_str().unwrap_or("")), - ), - ("Database backups to keep", &self.database_backups_to_keep.to_string()), - ("Database cache capacity (MB)", &self.db_cache_capacity_mb.to_string()), - ("Cache capacity modifier", &self.conduit_cache_capacity_modifier.to_string()), - ("PDU cache capacity", &self.pdu_cache_capacity.to_string()), - ("Auth chain cache capacity", &self.auth_chain_cache_capacity.to_string()), - ("Short eventid cache capacity", &self.shorteventid_cache_capacity.to_string()), - ("Eventid short cache capacity", &self.eventidshort_cache_capacity.to_string()), - ("Short statekey cache capacity", &self.shortstatekey_cache_capacity.to_string()), - ("Statekey short cache capacity", &self.statekeyshort_cache_capacity.to_string()), - ( - "Server visibility cache capacity", - &self.server_visibility_cache_capacity.to_string(), - ), - ( - "User visibility cache capacity", - &self.user_visibility_cache_capacity.to_string(), - ), - ("Stateinfo cache capacity", &self.stateinfo_cache_capacity.to_string()), - ( - "Roomid space hierarchy cache capacity", - &self.roomid_spacehierarchy_cache_capacity.to_string(), - ), - ("DNS cache entry limit", &self.dns_cache_entries.to_string()), - ("DNS minimum TTL", &self.dns_min_ttl.to_string()), - ("DNS minimum NXDOMAIN TTL", &self.dns_min_ttl_nxdomain.to_string()), - ("DNS attempts", &self.dns_attempts.to_string()), - ("DNS timeout", &self.dns_timeout.to_string()), - ("DNS fallback to TCP", &self.dns_tcp_fallback.to_string()), - ("DNS query over TCP only", &self.query_over_tcp_only.to_string()), - ("Query all nameservers", &self.query_all_nameservers.to_string()), - ("Maximum request size (bytes)", &self.max_request_size.to_string()), - ("Sender retry backoff limit", &self.sender_retry_backoff_limit.to_string()), - ("Request connect timeout", &self.request_conn_timeout.to_string()), - ("Request timeout", &self.request_timeout.to_string()), - ("Request total timeout", &self.request_total_timeout.to_string()), - ("Idle connections per host", &self.request_idle_per_host.to_string()), - ("Request pool idle timeout", &self.request_idle_timeout.to_string()), - ("Well_known connect timeout", &self.well_known_conn_timeout.to_string()), - ("Well_known timeout", &self.well_known_timeout.to_string()), - ("Federation timeout", &self.federation_timeout.to_string()), - ("Federation pool idle per host", &self.federation_idle_per_host.to_string()), - ("Federation pool idle timeout", &self.federation_idle_timeout.to_string()), - ("Sender timeout", &self.sender_timeout.to_string()), - ("Sender pool idle timeout", &self.sender_idle_timeout.to_string()), - ("Appservice timeout", &self.appservice_timeout.to_string()), - ("Appservice pool idle timeout", &self.appservice_idle_timeout.to_string()), - ("Pusher pool idle timeout", &self.pusher_idle_timeout.to_string()), - ("Allow registration", &self.allow_registration.to_string()), - ( - "Registration token", - if self.registration_token.is_some() { - "set" - } else { - "not set (open registration!)" - }, - ), - ( - "Allow guest registration (inherently false if allow registration is false)", - &self.allow_guest_registration.to_string(), - ), - ( - "Log guest registrations in admin room", - &self.log_guest_registrations.to_string(), - ), - ( - "Allow guests to auto join rooms", - &self.allow_guests_auto_join_rooms.to_string(), - ), - ("New user display name suffix", &self.new_user_displayname_suffix), - ("Allow encryption", &self.allow_encryption.to_string()), - ("Allow federation", &self.allow_federation.to_string()), - ( - "Allow incoming federated presence requests (updates)", - &self.allow_incoming_presence.to_string(), - ), - ( - "Allow outgoing federated presence requests (updates)", - &self.allow_outgoing_presence.to_string(), - ), - ( - "Allow local presence requests (updates)", - &self.allow_local_presence.to_string(), - ), - ( - "Allow incoming remote read receipts", - &self.allow_incoming_read_receipts.to_string(), - ), - ( - "Allow outgoing remote read receipts", - &self.allow_outgoing_read_receipts.to_string(), - ), - ( - "Block non-admin room invites (local and remote, admins can still send and receive invites)", - &self.block_non_admin_invites.to_string(), - ), - ("Enable admin escape commands", &self.admin_escape_commands.to_string()), - ("Allow outgoing federated typing", &self.allow_outgoing_typing.to_string()), - ("Allow incoming federated typing", &self.allow_incoming_typing.to_string()), - ( - "Incoming federated typing timeout", - &self.typing_federation_timeout_s.to_string(), - ), - ("Client typing timeout minimum", &self.typing_client_timeout_min_s.to_string()), - ("Client typing timeout maxmimum", &self.typing_client_timeout_max_s.to_string()), - ("Allow device name federation", &self.allow_device_name_federation.to_string()), - ( - "Allow incoming profile lookup federation requests", - &self.allow_profile_lookup_federation_requests.to_string(), - ), - ( - "Auto deactivate banned room join attempts", - &self.auto_deactivate_banned_room_attempts.to_string(), - ), - ("Notification push path", &self.notification_push_path), - ("Allow room creation", &self.allow_room_creation.to_string()), - ( - "Allow public room directory over federation", - &self.allow_public_room_directory_over_federation.to_string(), - ), - ( - "Allow public room directory without authentication", - &self.allow_public_room_directory_without_auth.to_string(), - ), - ( - "Lockdown public room directory (only allow admins to publish)", - &self.lockdown_public_room_directory.to_string(), - ), - ( - "JWT secret", - match self.jwt_secret { - Some(_) => "set", - None => "not set", - }, - ), - ( - "Trusted key servers", - &self - .trusted_servers - .iter() - .map(|server| server.host()) - .join(", "), - ), - ( - "Query Trusted Key Servers First", - &self.query_trusted_key_servers_first.to_string(), - ), - ("OpenID Token TTL", &self.openid_token_ttl.to_string()), - ( - "TURN username", - if self.turn_username.is_empty() { - "not set" - } else { - &self.turn_username - }, - ), - ("TURN password", { - if self.turn_password.is_empty() { - "not set" - } else { - "set" - } - }), - ("TURN secret", { - if self.turn_secret.is_empty() { - "not set" - } else { - "set" - } - }), - ("Turn TTL", &self.turn_ttl.to_string()), - ("Turn URIs", { - let mut lst = vec![]; - for item in self.turn_uris.iter().cloned().enumerate() { - let (_, uri): (usize, String) = item; - lst.push(uri); - } - &lst.join(", ") - }), - ("Auto Join Rooms", { - let mut lst = vec![]; - for room in &self.auto_join_rooms { - lst.push(room); - } - &lst.into_iter().join(", ") - }), - #[cfg(feature = "zstd_compression")] - ("Zstd HTTP Compression", &self.zstd_compression.to_string()), - #[cfg(feature = "gzip_compression")] - ("Gzip HTTP Compression", &self.gzip_compression.to_string()), - #[cfg(feature = "brotli_compression")] - ("Brotli HTTP Compression", &self.brotli_compression.to_string()), - ("RocksDB database LOG level", &self.rocksdb_log_level), - ("RocksDB database LOG to stderr", &self.rocksdb_log_stderr.to_string()), - ("RocksDB database LOG time-to-roll", &self.rocksdb_log_time_to_roll.to_string()), - ("RocksDB Max LOG Files", &self.rocksdb_max_log_files.to_string()), - ( - "RocksDB database max LOG file size", - &self.rocksdb_max_log_file_size.to_string(), - ), - ( - "RocksDB database optimize for spinning disks", - &self.rocksdb_optimize_for_spinning_disks.to_string(), - ), - ("RocksDB Direct-IO", &self.rocksdb_direct_io.to_string()), - ("RocksDB Parallelism Threads", &self.rocksdb_parallelism_threads.to_string()), - ("RocksDB Compression Algorithm", &self.rocksdb_compression_algo), - ("RocksDB Compression Level", &self.rocksdb_compression_level.to_string()), - ( - "RocksDB Bottommost Compression Level", - &self.rocksdb_bottommost_compression_level.to_string(), - ), - ( - "RocksDB Bottommost Level Compression", - &self.rocksdb_bottommost_compression.to_string(), - ), - ("RocksDB Recovery Mode", &self.rocksdb_recovery_mode.to_string()), - ("RocksDB Repair Mode", &self.rocksdb_repair.to_string()), - ("RocksDB Read-only Mode", &self.rocksdb_read_only.to_string()), - ( - "RocksDB Compaction Idle Priority", - &self.rocksdb_compaction_prio_idle.to_string(), - ), - ( - "RocksDB Compaction Idle IOPriority", - &self.rocksdb_compaction_ioprio_idle.to_string(), - ), - ("Media integrity checks on startup", &self.media_startup_check.to_string()), - ("Media compatibility filesystem links", &self.media_compat_file_link.to_string()), - ("Prevent Media Downloads From", { - let mut lst = vec![]; - for domain in &self.prevent_media_downloads_from { - lst.push(domain.host()); - } - &lst.join(", ") - }), - ("Forbidden Remote Server Names (\"Global\" ACLs)", { - let mut lst = vec![]; - for domain in &self.forbidden_remote_server_names { - lst.push(domain.host()); - } - &lst.join(", ") - }), - ("Forbidden Remote Room Directory Server Names", { - let mut lst = vec![]; - for domain in &self.forbidden_remote_room_directory_server_names { - lst.push(domain.host()); - } - &lst.join(", ") - }), - ("Outbound Request IP Range Denylist", { - let mut lst = vec![]; - for item in self.ip_range_denylist.iter().cloned().enumerate() { - let (_, ip): (usize, String) = item; - lst.push(ip); - } - &lst.join(", ") - }), - ("Forbidden usernames", { - &self.forbidden_usernames.patterns().iter().join(", ") - }), - ("Forbidden room aliases", { - &self.forbidden_alias_names.patterns().iter().join(", ") - }), - ( - "URL preview domain contains allowlist", - &self.url_preview_domain_contains_allowlist.join(", "), - ), - ( - "URL preview domain explicit allowlist", - &self.url_preview_domain_explicit_allowlist.join(", "), - ), - ( - "URL preview domain explicit denylist", - &self.url_preview_domain_explicit_denylist.join(", "), - ), - ( - "URL preview URL contains allowlist", - &self.url_preview_url_contains_allowlist.join(", "), - ), - ("URL preview maximum spider size", &self.url_preview_max_spider_size.to_string()), - ("URL preview check root domain", &self.url_preview_check_root_domain.to_string()), - ( - "Allow check for updates / announcements check", - &self.allow_check_for_updates.to_string(), - ), - ("Enable netburst on startup", &self.startup_netburst.to_string()), - #[cfg(feature = "sentry_telemetry")] - ("Sentry.io reporting and tracing", &self.sentry.to_string()), - #[cfg(feature = "sentry_telemetry")] - ("Sentry.io send server_name in logs", &self.sentry_send_server_name.to_string()), - #[cfg(feature = "sentry_telemetry")] - ("Sentry.io tracing sample rate", &self.sentry_traces_sample_rate.to_string()), - ("Sentry.io attach stacktrace", &self.sentry_attach_stacktrace.to_string()), - ("Sentry.io send panics", &self.sentry_send_panic.to_string()), - ("Sentry.io send errors", &self.sentry_send_error.to_string()), - ( - "Well-known server name", - self.well_known - .server - .as_ref() - .map_or("", |server| server.as_str()), - ), - ( - "Well-known client URL", - self.well_known - .client - .as_ref() - .map_or("", |url| url.as_str()), - ), - ( - "Well-known support email", - self.well_known - .support_email - .as_ref() - .map_or("", |str| str.as_ref()), - ), - ( - "Well-known support Matrix ID", - self.well_known - .support_mxid - .as_ref() - .map_or("", |mxid| mxid.as_str()), - ), - ( - "Well-known support role", - self.well_known - .support_role - .as_ref() - .map_or("", |role| role.as_str()), - ), - ( - "Well-known support page/URL", - self.well_known - .support_page - .as_ref() - .map_or("", |url| url.as_str()), - ), - ("Enable the tokio-console", &self.tokio_console.to_string()), - ]; + writeln!(f, "Active config values:\n\n").expect("wrote line to formatter stream"); + let mut line = |key: &str, val: &str| { + writeln!(f, "{key}: {val}").expect("wrote line to formatter stream"); + }; - let mut msg: String = "Active config values:\n\n".to_owned(); + line("Server name", self.server_name.host()); + line("Database backend", &self.database_backend); + line("Database path", &self.database_path.to_string_lossy()); + line( + "Database backup path", + self.database_backup_path + .as_ref() + .map_or("", |path| path.to_str().unwrap_or("")), + ); + line("Database backups to keep", &self.database_backups_to_keep.to_string()); + line("Database cache capacity (MB)", &self.db_cache_capacity_mb.to_string()); + line("Cache capacity modifier", &self.conduit_cache_capacity_modifier.to_string()); + line("PDU cache capacity", &self.pdu_cache_capacity.to_string()); + line("Auth chain cache capacity", &self.auth_chain_cache_capacity.to_string()); + line("Short eventid cache capacity", &self.shorteventid_cache_capacity.to_string()); + line("Eventid short cache capacity", &self.eventidshort_cache_capacity.to_string()); + line("Short statekey cache capacity", &self.shortstatekey_cache_capacity.to_string()); + line("Statekey short cache capacity", &self.statekeyshort_cache_capacity.to_string()); + line( + "Server visibility cache capacity", + &self.server_visibility_cache_capacity.to_string(), + ); + line( + "User visibility cache capacity", + &self.user_visibility_cache_capacity.to_string(), + ); + line("Stateinfo cache capacity", &self.stateinfo_cache_capacity.to_string()); + line( + "Roomid space hierarchy cache capacity", + &self.roomid_spacehierarchy_cache_capacity.to_string(), + ); + line("DNS cache entry limit", &self.dns_cache_entries.to_string()); + line("DNS minimum TTL", &self.dns_min_ttl.to_string()); + line("DNS minimum NXDOMAIN TTL", &self.dns_min_ttl_nxdomain.to_string()); + line("DNS attempts", &self.dns_attempts.to_string()); + line("DNS timeout", &self.dns_timeout.to_string()); + line("DNS fallback to TCP", &self.dns_tcp_fallback.to_string()); + line("DNS query over TCP only", &self.query_over_tcp_only.to_string()); + line("Query all nameservers", &self.query_all_nameservers.to_string()); + line("Maximum request size (bytes)", &self.max_request_size.to_string()); + line("Sender retry backoff limit", &self.sender_retry_backoff_limit.to_string()); + line("Request connect timeout", &self.request_conn_timeout.to_string()); + line("Request timeout", &self.request_timeout.to_string()); + line("Request total timeout", &self.request_total_timeout.to_string()); + line("Idle connections per host", &self.request_idle_per_host.to_string()); + line("Request pool idle timeout", &self.request_idle_timeout.to_string()); + line("Well_known connect timeout", &self.well_known_conn_timeout.to_string()); + line("Well_known timeout", &self.well_known_timeout.to_string()); + line("Federation timeout", &self.federation_timeout.to_string()); + line("Federation pool idle per host", &self.federation_idle_per_host.to_string()); + line("Federation pool idle timeout", &self.federation_idle_timeout.to_string()); + line("Sender timeout", &self.sender_timeout.to_string()); + line("Sender pool idle timeout", &self.sender_idle_timeout.to_string()); + line("Appservice timeout", &self.appservice_timeout.to_string()); + line("Appservice pool idle timeout", &self.appservice_idle_timeout.to_string()); + line("Pusher pool idle timeout", &self.pusher_idle_timeout.to_string()); + line("Allow registration", &self.allow_registration.to_string()); + line( + "Registration token", + if self.registration_token.is_some() { + "set" + } else { + "not set (open registration!)" + }, + ); + line( + "Allow guest registration (inherently false if allow registration is false)", + &self.allow_guest_registration.to_string(), + ); + line( + "Log guest registrations in admin room", + &self.log_guest_registrations.to_string(), + ); + line( + "Allow guests to auto join rooms", + &self.allow_guests_auto_join_rooms.to_string(), + ); + line("New user display name suffix", &self.new_user_displayname_suffix); + line("Allow encryption", &self.allow_encryption.to_string()); + line("Allow federation", &self.allow_federation.to_string()); + line( + "Allow incoming federated presence requests (updates)", + &self.allow_incoming_presence.to_string(), + ); + line( + "Allow outgoing federated presence requests (updates)", + &self.allow_outgoing_presence.to_string(), + ); + line( + "Allow local presence requests (updates)", + &self.allow_local_presence.to_string(), + ); + line( + "Allow incoming remote read receipts", + &self.allow_incoming_read_receipts.to_string(), + ); + line( + "Allow outgoing remote read receipts", + &self.allow_outgoing_read_receipts.to_string(), + ); + line( + "Block non-admin room invites (local and remote, admins can still send and receive invites)", + &self.block_non_admin_invites.to_string(), + ); + line("Enable admin escape commands", &self.admin_escape_commands.to_string()); + line("Allow outgoing federated typing", &self.allow_outgoing_typing.to_string()); + line("Allow incoming federated typing", &self.allow_incoming_typing.to_string()); + line( + "Incoming federated typing timeout", + &self.typing_federation_timeout_s.to_string(), + ); + line("Client typing timeout minimum", &self.typing_client_timeout_min_s.to_string()); + line("Client typing timeout maxmimum", &self.typing_client_timeout_max_s.to_string()); + line("Allow device name federation", &self.allow_device_name_federation.to_string()); + line( + "Allow incoming profile lookup federation requests", + &self.allow_profile_lookup_federation_requests.to_string(), + ); + line( + "Auto deactivate banned room join attempts", + &self.auto_deactivate_banned_room_attempts.to_string(), + ); + line("Notification push path", &self.notification_push_path); + line("Allow room creation", &self.allow_room_creation.to_string()); + line( + "Allow public room directory over federation", + &self.allow_public_room_directory_over_federation.to_string(), + ); + line( + "Allow public room directory without authentication", + &self.allow_public_room_directory_without_auth.to_string(), + ); + line( + "Lockdown public room directory (only allow admins to publish)", + &self.lockdown_public_room_directory.to_string(), + ); + line( + "JWT secret", + match self.jwt_secret { + Some(_) => "set", + None => "not set", + }, + ); + line( + "Trusted key servers", + &self + .trusted_servers + .iter() + .map(|server| server.host()) + .join(", "), + ); + line( + "Query Trusted Key Servers First", + &self.query_trusted_key_servers_first.to_string(), + ); + line("OpenID Token TTL", &self.openid_token_ttl.to_string()); + line( + "TURN username", + if self.turn_username.is_empty() { + "not set" + } else { + &self.turn_username + }, + ); + line("TURN password", { + if self.turn_password.is_empty() { + "not set" + } else { + "set" + } + }); + line("TURN secret", { + if self.turn_secret.is_empty() { + "not set" + } else { + "set" + } + }); + line("Turn TTL", &self.turn_ttl.to_string()); + line("Turn URIs", { + let mut lst = vec![]; + for item in self.turn_uris.iter().cloned().enumerate() { + let (_, uri): (usize, String) = item; + lst.push(uri); + } + &lst.join(", ") + }); + line("Auto Join Rooms", { + let mut lst = vec![]; + for room in &self.auto_join_rooms { + lst.push(room); + } + &lst.into_iter().join(", ") + }); + #[cfg(feature = "zstd_compression")] + line("Zstd HTTP Compression", &self.zstd_compression.to_string()); + #[cfg(feature = "gzip_compression")] + line("Gzip HTTP Compression", &self.gzip_compression.to_string()); + #[cfg(feature = "brotli_compression")] + line("Brotli HTTP Compression", &self.brotli_compression.to_string()); + line("RocksDB database LOG level", &self.rocksdb_log_level); + line("RocksDB database LOG to stderr", &self.rocksdb_log_stderr.to_string()); + line("RocksDB database LOG time-to-roll", &self.rocksdb_log_time_to_roll.to_string()); + line("RocksDB Max LOG Files", &self.rocksdb_max_log_files.to_string()); + line( + "RocksDB database max LOG file size", + &self.rocksdb_max_log_file_size.to_string(), + ); + line( + "RocksDB database optimize for spinning disks", + &self.rocksdb_optimize_for_spinning_disks.to_string(), + ); + line("RocksDB Direct-IO", &self.rocksdb_direct_io.to_string()); + line("RocksDB Parallelism Threads", &self.rocksdb_parallelism_threads.to_string()); + line("RocksDB Compression Algorithm", &self.rocksdb_compression_algo); + line("RocksDB Compression Level", &self.rocksdb_compression_level.to_string()); + line( + "RocksDB Bottommost Compression Level", + &self.rocksdb_bottommost_compression_level.to_string(), + ); + line( + "RocksDB Bottommost Level Compression", + &self.rocksdb_bottommost_compression.to_string(), + ); + line("RocksDB Recovery Mode", &self.rocksdb_recovery_mode.to_string()); + line("RocksDB Repair Mode", &self.rocksdb_repair.to_string()); + line("RocksDB Read-only Mode", &self.rocksdb_read_only.to_string()); + line( + "RocksDB Compaction Idle Priority", + &self.rocksdb_compaction_prio_idle.to_string(), + ); + line( + "RocksDB Compaction Idle IOPriority", + &self.rocksdb_compaction_ioprio_idle.to_string(), + ); + line("Media integrity checks on startup", &self.media_startup_check.to_string()); + line("Media compatibility filesystem links", &self.media_compat_file_link.to_string()); + line("Prevent Media Downloads From", { + let mut lst = vec![]; + for domain in &self.prevent_media_downloads_from { + lst.push(domain.host()); + } + &lst.join(", ") + }); + line("Forbidden Remote Server Names (\"Global\" ACLs)", { + let mut lst = vec![]; + for domain in &self.forbidden_remote_server_names { + lst.push(domain.host()); + } + &lst.join(", ") + }); + line("Forbidden Remote Room Directory Server Names", { + let mut lst = vec![]; + for domain in &self.forbidden_remote_room_directory_server_names { + lst.push(domain.host()); + } + &lst.join(", ") + }); + line("Outbound Request IP Range Denylist", { + let mut lst = vec![]; + for item in self.ip_range_denylist.iter().cloned().enumerate() { + let (_, ip): (usize, String) = item; + lst.push(ip); + } + &lst.join(", ") + }); + line("Forbidden usernames", { + &self.forbidden_usernames.patterns().iter().join(", ") + }); + line("Forbidden room aliases", { + &self.forbidden_alias_names.patterns().iter().join(", ") + }); + line( + "URL preview domain contains allowlist", + &self.url_preview_domain_contains_allowlist.join(", "), + ); + line( + "URL preview domain explicit allowlist", + &self.url_preview_domain_explicit_allowlist.join(", "), + ); + line( + "URL preview domain explicit denylist", + &self.url_preview_domain_explicit_denylist.join(", "), + ); + line( + "URL preview URL contains allowlist", + &self.url_preview_url_contains_allowlist.join(", "), + ); + line("URL preview maximum spider size", &self.url_preview_max_spider_size.to_string()); + line("URL preview check root domain", &self.url_preview_check_root_domain.to_string()); + line( + "Allow check for updates / announcements check", + &self.allow_check_for_updates.to_string(), + ); + line("Enable netburst on startup", &self.startup_netburst.to_string()); + #[cfg(feature = "sentry_telemetry")] + line("Sentry.io reporting and tracing", &self.sentry.to_string()); + #[cfg(feature = "sentry_telemetry")] + line("Sentry.io send server_name in logs", &self.sentry_send_server_name.to_string()); + #[cfg(feature = "sentry_telemetry")] + line("Sentry.io tracing sample rate", &self.sentry_traces_sample_rate.to_string()); + line("Sentry.io attach stacktrace", &self.sentry_attach_stacktrace.to_string()); + line("Sentry.io send panics", &self.sentry_send_panic.to_string()); + line("Sentry.io send errors", &self.sentry_send_error.to_string()); + line( + "Well-known server name", + self.well_known + .server + .as_ref() + .map_or("", |server| server.as_str()), + ); + line( + "Well-known client URL", + self.well_known + .client + .as_ref() + .map_or("", |url| url.as_str()), + ); + line( + "Well-known support email", + self.well_known + .support_email + .as_ref() + .map_or("", |str| str.as_ref()), + ); + line( + "Well-known support Matrix ID", + self.well_known + .support_mxid + .as_ref() + .map_or("", |mxid| mxid.as_str()), + ); + line( + "Well-known support role", + self.well_known + .support_role + .as_ref() + .map_or("", |role| role.as_str()), + ); + line( + "Well-known support page/URL", + self.well_known + .support_page + .as_ref() + .map_or("", |url| url.as_str()), + ); + line("Enable the tokio-console", &self.tokio_console.to_string()); - for line in lines.into_iter().enumerate() { - writeln!(msg, "{}: {}", line.1 .0, line.1 .1).expect("should be able to write to string buffer"); - } - - write!(f, "{msg}") + Ok(()) } } From 899b79873e51b5746ca466f3b7bdcaa864d036be Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 11 Jul 2024 05:03:16 +0000 Subject: [PATCH 103/158] propagate errors from tracing init Signed-off-by: Jason Volk --- src/main/server.rs | 2 +- src/main/tracing.rs | 57 +++++++++++++++++---------------------------- 2 files changed, 22 insertions(+), 37 deletions(-) diff --git a/src/main/server.rs b/src/main/server.rs index f72b3ef3..73c06f0c 100644 --- a/src/main/server.rs +++ b/src/main/server.rs @@ -27,7 +27,7 @@ impl Server { #[cfg(feature = "sentry_telemetry")] let sentry_guard = crate::sentry::init(&config); - let (tracing_reload_handle, tracing_flame_guard, capture) = crate::tracing::init(&config); + let (tracing_reload_handle, tracing_flame_guard, capture) = crate::tracing::init(&config)?; config.check()?; diff --git a/src/main/tracing.rs b/src/main/tracing.rs index 78c35c66..31870661 100644 --- a/src/main/tracing.rs +++ b/src/main/tracing.rs @@ -1,10 +1,10 @@ use std::sync::Arc; use conduit::{ - config, config::Config, debug_warn, log::{capture, LogLevelReloadHandles}, + Error, Result, }; use tracing_subscriber::{layer::SubscriberExt, reload, EnvFilter, Layer, Registry}; @@ -14,26 +14,21 @@ pub(crate) type TracingFlameGuard = Option (LogLevelReloadHandles, TracingFlameGuard, Arc) { +pub(crate) fn init(config: &Config) -> Result<(LogLevelReloadHandles, TracingFlameGuard, Arc)> { let reload_handles = LogLevelReloadHandles::default(); - let fmt_layer = tracing_subscriber::fmt::Layer::new(); - let filter_layer = match EnvFilter::try_new(&config.log) { - Ok(s) => s, - Err(e) => { - eprintln!("It looks like your config is invalid. The following error occured while parsing it: {e}"); - EnvFilter::try_new(config::default_log()).expect("failed to set default EnvFilter") - }, - }; - - let (fmt_reload_filter, fmt_reload_handle) = reload::Layer::new(filter_layer.clone()); - reload_handles.add("console", Box::new(fmt_reload_handle)); - - let subscriber = Registry::default().with(fmt_layer.with_filter(fmt_reload_filter)); + let console_filter = + EnvFilter::try_new(&config.log).map_err(|e| Error::BadConfig(format!("in the 'log' setting: {e}.")))?; + let console_layer = tracing_subscriber::fmt::Layer::new(); + let (console_reload_filter, console_reload_handle) = reload::Layer::new(console_filter.clone()); + reload_handles.add("console", Box::new(console_reload_handle)); let cap_state = Arc::new(capture::State::new()); let cap_layer = capture::Layer::new(&cap_state); - let subscriber = subscriber.with(cap_layer); + + let subscriber = Registry::default() + .with(console_layer.with_filter(console_reload_filter)) + .with(cap_layer); #[cfg(feature = "sentry_telemetry")] let subscriber = { @@ -46,16 +41,10 @@ pub(crate) fn init(config: &Config) -> (LogLevelReloadHandles, TracingFlameGuard #[cfg(feature = "perf_measurements")] let (subscriber, flame_guard) = { let (flame_layer, flame_guard) = if config.tracing_flame { - let flame_filter = match EnvFilter::try_new(&config.tracing_flame_filter) { - Ok(flame_filter) => flame_filter, - Err(e) => panic!("tracing_flame_filter config value is invalid: {e}"), - }; - - let (flame_layer, flame_guard) = - match tracing_flame::FlameLayer::with_file(&config.tracing_flame_output_path) { - Ok(ok) => ok, - Err(e) => panic!("failed to initialize tracing-flame: {e}"), - }; + let flame_filter = EnvFilter::try_new(&config.tracing_flame_filter) + .map_err(|e| Error::BadConfig(format!("in the 'tracing_flame_filter' setting: {e}.")))?; + let (flame_layer, flame_guard) = tracing_flame::FlameLayer::with_file(&config.tracing_flame_output_path) + .map_err(|e| Error::BadConfig(format!("in the 'tracing_flame_output_path' setting: {e}.")))?; let flame_layer = flame_layer .with_empty_samples(false) .with_filter(flame_filter); @@ -64,22 +53,18 @@ pub(crate) fn init(config: &Config) -> (LogLevelReloadHandles, TracingFlameGuard (None, None) }; - let jaeger_layer = if config.allow_jaeger { + let jaeger_layer = config.allow_jaeger.then(|| { opentelemetry::global::set_text_map_propagator(opentelemetry_jaeger::Propagator::new()); let tracer = opentelemetry_jaeger::new_agent_pipeline() .with_auto_split_batch(true) .with_service_name("conduwuit") .install_batch(opentelemetry_sdk::runtime::Tokio) - .unwrap(); + .expect("jaeger agent pipeline"); let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); - - let (jaeger_reload_filter, jaeger_reload_handle) = reload::Layer::new(filter_layer.clone()); + let (jaeger_reload_filter, jaeger_reload_handle) = reload::Layer::new(console_filter.clone()); reload_handles.add("jaeger", Box::new(jaeger_reload_handle)); Some(telemetry.with_filter(jaeger_reload_filter)) - } else { - None - }; - + }); let subscriber = subscriber.with(flame_layer).with(jaeger_layer); (subscriber, flame_guard) }; @@ -101,7 +86,7 @@ pub(crate) fn init(config: &Config) -> (LogLevelReloadHandles, TracingFlameGuard .spawn(); set_global_default(subscriber.with(console_layer)); - return ret; + return Ok(ret); } set_global_default(subscriber); @@ -112,7 +97,7 @@ pub(crate) fn init(config: &Config) -> (LogLevelReloadHandles, TracingFlameGuard debug_warn!("{console_disabled_reason}"); } - ret + Ok(ret) } fn tokio_console_enabled(config: &Config) -> (bool, &'static str) { From 5570220c8901d6a423a1c2ea4d86f8ed5d8b1ba4 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 11 Jul 2024 05:03:41 +0000 Subject: [PATCH 104/158] use separate but configurable envfilter for sentry Signed-off-by: Jason Volk --- src/core/config/mod.rs | 5 +++++ src/main/tracing.rs | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index df13ad1f..4eefddcf 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -362,6 +362,8 @@ pub struct Config { pub sentry_send_panic: bool, #[serde(default = "true_fn")] pub sentry_send_error: bool, + #[serde(default = "default_sentry_filter")] + pub sentry_filter: String, #[serde(default)] pub tokio_console: bool, @@ -831,6 +833,7 @@ impl fmt::Display for Config { line("Sentry.io attach stacktrace", &self.sentry_attach_stacktrace.to_string()); line("Sentry.io send panics", &self.sentry_send_panic.to_string()); line("Sentry.io send errors", &self.sentry_send_error.to_string()); + line("Sentry.io tracing filter", &self.sentry_filter); line( "Well-known server name", self.well_known @@ -1075,4 +1078,6 @@ fn default_sentry_endpoint() -> Option { fn default_sentry_traces_sample_rate() -> f32 { 0.15 } +fn default_sentry_filter() -> String { "info".to_owned() } + fn default_startup_netburst_keep() -> i64 { 50 } diff --git a/src/main/tracing.rs b/src/main/tracing.rs index 31870661..ccfe188d 100644 --- a/src/main/tracing.rs +++ b/src/main/tracing.rs @@ -32,8 +32,10 @@ pub(crate) fn init(config: &Config) -> Result<(LogLevelReloadHandles, TracingFla #[cfg(feature = "sentry_telemetry")] let subscriber = { + let sentry_filter = EnvFilter::try_new(&config.sentry_filter) + .map_err(|e| Error::BadConfig(format!("in the 'sentry_filter' setting: {e}.")))?; let sentry_layer = sentry_tracing::layer(); - let (sentry_reload_filter, sentry_reload_handle) = reload::Layer::new(filter_layer.clone()); + let (sentry_reload_filter, sentry_reload_handle) = reload::Layer::new(sentry_filter); reload_handles.add("sentry", Box::new(sentry_reload_handle)); subscriber.with(sentry_layer.with_filter(sentry_reload_filter)) }; From a35b6cbfdd8cbea73056b94652bc43807d0f1521 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 11 Jul 2024 05:26:43 +0000 Subject: [PATCH 105/158] use separate configurable jaeger envfilter Signed-off-by: Jason Volk --- src/core/config/mod.rs | 11 +++++++++++ src/main/tracing.rs | 5 ++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 4eefddcf..aadfe564 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -183,6 +183,9 @@ pub struct Config { #[serde(default)] #[cfg(feature = "perf_measurements")] pub allow_jaeger: bool, + #[serde(default = "default_jaeger_filter")] + #[cfg(feature = "perf_measurements")] + pub jaeger_filter: String, #[serde(default)] #[cfg(feature = "perf_measurements")] pub tracing_flame: bool, @@ -979,6 +982,14 @@ fn default_max_fetch_prev_events() -> u16 { 100_u16 } #[cfg(feature = "perf_measurements")] fn default_tracing_flame_filter() -> String { "trace,h2=off".to_owned() } +#[cfg(feature = "perf_measurements")] +fn default_jaeger_filter() -> String { + cfg!(debug_assertions) + .then_some("trace,h2=off") + .unwrap_or("info") + .to_owned() +} + #[cfg(feature = "perf_measurements")] fn default_tracing_flame_output_path() -> String { "./tracing.folded".to_owned() } diff --git a/src/main/tracing.rs b/src/main/tracing.rs index ccfe188d..67977088 100644 --- a/src/main/tracing.rs +++ b/src/main/tracing.rs @@ -55,6 +55,8 @@ pub(crate) fn init(config: &Config) -> Result<(LogLevelReloadHandles, TracingFla (None, None) }; + let jaeger_filter = EnvFilter::try_new(&config.jaeger_filter) + .map_err(|e| Error::BadConfig(format!("in the 'jaeger_filter' setting: {e}.")))?; let jaeger_layer = config.allow_jaeger.then(|| { opentelemetry::global::set_text_map_propagator(opentelemetry_jaeger::Propagator::new()); let tracer = opentelemetry_jaeger::new_agent_pipeline() @@ -63,10 +65,11 @@ pub(crate) fn init(config: &Config) -> Result<(LogLevelReloadHandles, TracingFla .install_batch(opentelemetry_sdk::runtime::Tokio) .expect("jaeger agent pipeline"); let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); - let (jaeger_reload_filter, jaeger_reload_handle) = reload::Layer::new(console_filter.clone()); + let (jaeger_reload_filter, jaeger_reload_handle) = reload::Layer::new(jaeger_filter.clone()); reload_handles.add("jaeger", Box::new(jaeger_reload_handle)); Some(telemetry.with_filter(jaeger_reload_filter)) }); + let subscriber = subscriber.with(flame_layer).with(jaeger_layer); (subscriber, flame_guard) }; From 03d890cd498e8ce55d9e86d38f0b0525c18ce61a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 11 Jul 2024 08:54:33 +0000 Subject: [PATCH 106/158] move admin tests into unit; fix Signed-off-by: Jason Volk --- src/admin/handler.rs | 1 - src/admin/mod.rs | 28 +--------------------------- src/admin/tests.rs | 26 ++++++++++++++++++++++++++ 3 files changed, 27 insertions(+), 28 deletions(-) create mode 100644 src/admin/tests.rs diff --git a/src/admin/handler.rs b/src/admin/handler.rs index 2d1f96b5..95c7ed41 100644 --- a/src/admin/handler.rs +++ b/src/admin/handler.rs @@ -24,7 +24,6 @@ use crate::{ }; pub(crate) const PAGE_SIZE: usize = 100; -#[cfg_attr(test, derive(Debug))] #[derive(Parser)] #[command(name = "admin", version = env!("CARGO_PKG_VERSION"))] pub(crate) enum AdminCommand { diff --git a/src/admin/mod.rs b/src/admin/mod.rs index 6a47bc74..14856811 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -9,6 +9,7 @@ pub(crate) mod media; pub(crate) mod query; pub(crate) mod room; pub(crate) mod server; +mod tests; pub(crate) mod user; pub(crate) mod utils; @@ -53,30 +54,3 @@ pub async fn fini() { .expect("locked for writing") .take(); } - -#[cfg(test)] -mod test { - use clap::Parser; - - use crate::handler::AdminCommand; - - #[test] - fn get_help_short() { get_help_inner("-h"); } - - #[test] - fn get_help_long() { get_help_inner("--help"); } - - #[test] - fn get_help_subcommand() { get_help_inner("help"); } - - fn get_help_inner(input: &str) { - let error = AdminCommand::try_parse_from(["argv[0] doesn't matter", input]) - .unwrap_err() - .to_string(); - - // Search for a handful of keywords that suggest the help printed properly - assert!(error.contains("Usage:")); - assert!(error.contains("Commands:")); - assert!(error.contains("Options:")); - } -} diff --git a/src/admin/tests.rs b/src/admin/tests.rs new file mode 100644 index 00000000..69ccd896 --- /dev/null +++ b/src/admin/tests.rs @@ -0,0 +1,26 @@ +#![cfg(test)] + +#[test] +fn get_help_short() { get_help_inner("-h"); } + +#[test] +fn get_help_long() { get_help_inner("--help"); } + +#[test] +fn get_help_subcommand() { get_help_inner("help"); } + +fn get_help_inner(input: &str) { + use clap::Parser; + + use crate::handler::AdminCommand; + + let Err(error) = AdminCommand::try_parse_from(["argv[0] doesn't matter", input]) else { + panic!("no error!"); + }; + + let error = error.to_string(); + // Search for a handful of keywords that suggest the help printed properly + assert!(error.contains("Usage:")); + assert!(error.contains("Commands:")); + assert!(error.contains("Options:")); +} From c111d2e39536672224e4a2efe19d674e5ac7c6a0 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 11 Jul 2024 21:00:30 +0000 Subject: [PATCH 107/158] abstract service worker pattern; restart on panic. Signed-off-by: Jason Volk --- src/router/run.rs | 6 +- src/service/admin/mod.rs | 64 +++-------- src/service/mod.rs | 6 +- src/service/presence/mod.rs | 57 +++------ src/service/sending/mod.rs | 50 +------- src/service/sending/sender.rs | 39 ++++--- src/service/service.rs | 17 +-- src/service/services.rs | 210 +++++++++++++++++++++++++--------- 8 files changed, 233 insertions(+), 216 deletions(-) diff --git a/src/router/run.rs b/src/router/run.rs index 3e09823a..02cec781 100644 --- a/src/router/run.rs +++ b/src/router/run.rs @@ -11,7 +11,6 @@ extern crate conduit_service as service; use std::sync::atomic::Ordering; use conduit::{debug_info, trace, Error, Result, Server}; -use service::services; use crate::{layers, serve}; @@ -50,7 +49,6 @@ pub(crate) async fn start(server: Arc) -> Result<(), Error> { debug!("Starting..."); service::init(&server).await?; - services().start().await?; #[cfg(feature = "systemd")] sd_notify::notify(true, &[sd_notify::NotifyState::Ready]).expect("failed to notify systemd of ready state"); @@ -66,9 +64,7 @@ pub(crate) async fn stop(_server: Arc) -> Result<(), Error> { // Wait for all completions before dropping or we'll lose them to the module // unload and explode. - services().stop().await; - // Deactivate services(). Any further use will panic the caller. - service::fini(); + service::fini().await; debug!("Cleaning up..."); diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index e9729d2d..41019cd1 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -21,17 +21,13 @@ use ruma::{ OwnedEventId, OwnedRoomId, RoomId, UserId, }; use serde_json::value::to_raw_value; -use tokio::{ - sync::{Mutex, RwLock}, - task::JoinHandle, -}; +use tokio::sync::{Mutex, RwLock}; use crate::{pdu::PduBuilder, rooms::state::RoomMutexGuard, services, user_is_local, PduEvent}; pub struct Service { sender: Sender, receiver: Mutex>, - handler_join: Mutex>>, pub handle: RwLock>, pub complete: StdRwLock>, #[cfg(feature = "console")] @@ -59,7 +55,6 @@ impl crate::Service for Service { Ok(Arc::new(Self { sender, receiver: Mutex::new(receiver), - handler_join: Mutex::new(None), handle: RwLock::new(None), complete: StdRwLock::new(None), #[cfg(feature = "console")] @@ -67,16 +62,25 @@ impl crate::Service for Service { })) } - async fn start(self: Arc) -> Result<()> { - let self_ = Arc::clone(&self); - let handle = services().server.runtime().spawn(async move { - self_ - .handler() - .await - .expect("Failed to initialize admin room handler"); - }); + async fn worker(self: Arc) -> Result<()> { + let receiver = self.receiver.lock().await; + let mut signals = services().server.signal.subscribe(); + loop { + tokio::select! { + command = receiver.recv_async() => match command { + Ok(command) => self.handle_command(command).await, + Err(_) => break, + }, + sig = signals.recv() => match sig { + Ok(sig) => self.handle_signal(sig).await, + Err(_) => continue, + }, + } + } - _ = self.handler_join.lock().await.insert(handle); + //TODO: not unwind safe + #[cfg(feature = "console")] + self.console.close().await; Ok(()) } @@ -90,19 +94,6 @@ impl crate::Service for Service { } } - async fn stop(&self) { - self.interrupt(); - - #[cfg(feature = "console")] - self.console.close().await; - - if let Some(handler_join) = self.handler_join.lock().await.take() { - if let Err(e) = handler_join.await { - error!("Failed to shutdown: {e:?}"); - } - } - } - fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } @@ -149,23 +140,6 @@ impl Service { self.sender.send_async(message).await.expect("message sent"); } - async fn handler(self: &Arc) -> Result<()> { - let receiver = self.receiver.lock().await; - let mut signals = services().server.signal.subscribe(); - loop { - tokio::select! { - command = receiver.recv_async() => match command { - Ok(command) => self.handle_command(command).await, - Err(_) => return Ok(()), - }, - sig = signals.recv() => match sig { - Ok(sig) => self.handle_signal(sig).await, - Err(_) => continue, - }, - } - } - } - async fn handle_signal(&self, #[allow(unused_variables)] sig: &'static str) { #[cfg(feature = "console")] self.console.handle_signal(sig).await; diff --git a/src/service/mod.rs b/src/service/mod.rs index 4b19073d..15c4cc35 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -42,10 +42,12 @@ pub async fn init(server: &Arc) -> Result<()> { let s = Box::new(Services::build(server.clone(), d)?); _ = SERVICES.write().expect("write locked").insert(Box::leak(s)); - Ok(()) + services().start().await } -pub fn fini() { +pub async fn fini() { + services().stop().await; + // Deactivate services(). Any further use will panic the caller. let s = SERVICES .write() diff --git a/src/service/presence/mod.rs b/src/service/presence/mod.rs index f5400379..254304ba 100644 --- a/src/service/presence/mod.rs +++ b/src/service/presence/mod.rs @@ -12,7 +12,7 @@ use ruma::{ OwnedUserId, UInt, UserId, }; use serde::{Deserialize, Serialize}; -use tokio::{sync::Mutex, task::JoinHandle, time::sleep}; +use tokio::{sync::Mutex, time::sleep}; use crate::{services, user_is_local}; @@ -77,7 +77,6 @@ pub struct Service { pub db: Data, pub timer_sender: loole::Sender<(OwnedUserId, Duration)>, timer_receiver: Mutex>, - handler_join: Mutex>>, timeout_remote_users: bool, idle_timeout: u64, offline_timeout: u64, @@ -94,34 +93,26 @@ impl crate::Service for Service { db: Data::new(args.db), timer_sender, timer_receiver: Mutex::new(timer_receiver), - handler_join: Mutex::new(None), timeout_remote_users: config.presence_timeout_remote_users, idle_timeout: checked!(idle_timeout_s * 1_000)?, offline_timeout: checked!(offline_timeout_s * 1_000)?, })) } - async fn start(self: Arc) -> Result<()> { - //TODO: if self.globals.config.allow_local_presence { return; } - - let self_ = Arc::clone(&self); - let handle = services().server.runtime().spawn(async move { - self_ - .handler() - .await - .expect("Failed to start presence handler"); - }); - - _ = self.handler_join.lock().await.insert(handle); - - Ok(()) - } - - async fn stop(&self) { - self.interrupt(); - if let Some(handler_join) = self.handler_join.lock().await.take() { - if let Err(e) = handler_join.await { - error!("Failed to shutdown: {e:?}"); + async fn worker(self: Arc) -> Result<()> { + let mut presence_timers = FuturesUnordered::new(); + let receiver = self.timer_receiver.lock().await; + loop { + debug_assert!(!receiver.is_closed(), "channel error"); + tokio::select! { + Some(user_id) = presence_timers.next() => self.process_presence_timer(&user_id)?, + event = receiver.recv_async() => match event { + Err(_e) => return Ok(()), + Ok((user_id, timeout)) => { + debug!("Adding timer {}: {user_id} timeout:{timeout:?}", presence_timers.len()); + presence_timers.push(presence_timer(user_id, timeout)); + }, + }, } } } @@ -219,24 +210,6 @@ impl Service { self.db.presence_since(since) } - async fn handler(&self) -> Result<()> { - let mut presence_timers = FuturesUnordered::new(); - let receiver = self.timer_receiver.lock().await; - loop { - debug_assert!(!receiver.is_closed(), "channel error"); - tokio::select! { - Some(user_id) = presence_timers.next() => self.process_presence_timer(&user_id)?, - event = receiver.recv_async() => match event { - Err(_e) => return Ok(()), - Ok((user_id, timeout)) => { - debug!("Adding timer {}: {user_id} timeout:{timeout:?}", presence_timers.len()); - presence_timers.push(presence_timer(user_id, timeout)); - }, - }, - } - } - } - fn process_presence_timer(&self, user_id: &OwnedUserId) -> Result<()> { let mut presence_state = PresenceState::Offline; let mut last_active_ago = None; diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index d7a9c0fc..eb708fcf 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -4,29 +4,26 @@ mod resolve; mod send; mod sender; -use std::{fmt::Debug, sync::Arc}; +use std::fmt::Debug; -use async_trait::async_trait; use conduit::{Error, Result}; -use data::Data; pub use resolve::{resolve_actual_dest, CachedDest, CachedOverride, FedDest}; use ruma::{ api::{appservice::Registration, OutgoingRequest}, OwnedServerName, OwnedUserId, RoomId, ServerName, UserId, }; pub use sender::convert_to_outgoing_federation_event; -use tokio::{sync::Mutex, task::JoinHandle}; -use tracing::{error, warn}; +use tokio::sync::Mutex; +use tracing::warn; use crate::{server_is_ours, services}; pub struct Service { - pub db: Data, + pub db: data::Data, /// The state for a given state hash. sender: loole::Sender, receiver: Mutex>, - handler_join: Mutex>>, startup_netburst: bool, startup_netburst_keep: i64, } @@ -53,45 +50,6 @@ pub enum SendingEvent { Flush, // none } -#[async_trait] -impl crate::Service for Service { - fn build(args: crate::Args<'_>) -> Result> { - let config = &args.server.config; - let (sender, receiver) = loole::unbounded(); - Ok(Arc::new(Self { - db: Data::new(args.db.clone()), - sender, - receiver: Mutex::new(receiver), - handler_join: Mutex::new(None), - startup_netburst: config.startup_netburst, - startup_netburst_keep: config.startup_netburst_keep, - })) - } - - async fn start(self: Arc) -> Result<()> { - self.start_handler().await; - - Ok(()) - } - - async fn stop(&self) { - self.interrupt(); - if let Some(handler_join) = self.handler_join.lock().await.take() { - if let Err(e) = handler_join.await { - error!("Failed to shutdown: {e:?}"); - } - } - } - - fn interrupt(&self) { - if !self.sender.is_closed() { - self.sender.close(); - } - } - - fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } -} - impl Service { #[tracing::instrument(skip(self, pdu_id, user, pushkey), level = "debug")] pub fn send_pdu_push(&self, pdu_id: &[u8], user: &UserId, pushkey: String) -> Result<()> { diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 0fb0d9dc..cfd5b4bc 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -6,6 +6,7 @@ use std::{ time::Instant, }; +use async_trait::async_trait; use base64::{engine::general_purpose, Engine as _}; use conduit::{debug, error, utils::math::continue_exponential_backoff_secs, warn}; use federation::transactions::send_transaction_message; @@ -23,8 +24,9 @@ use ruma::{ ServerName, UInt, }; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; +use tokio::sync::Mutex; -use super::{appservice, send, Destination, Msg, SendingEvent, Service}; +use super::{appservice, data::Data, send, Destination, Msg, SendingEvent, Service}; use crate::{presence::Presence, services, user_is_local, utils::calculate_hash, Error, Result}; #[derive(Debug)] @@ -43,21 +45,22 @@ type CurTransactionStatus = HashMap; const DEQUEUE_LIMIT: usize = 48; const SELECT_EDU_LIMIT: usize = 16; -impl Service { - pub async fn start_handler(self: &Arc) { - let self_ = Arc::clone(self); - let handle = services().server.runtime().spawn(async move { - self_ - .handler() - .await - .expect("Failed to start sending handler"); - }); - - _ = self.handler_join.lock().await.insert(handle); +#[async_trait] +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + let config = &args.server.config; + let (sender, receiver) = loole::unbounded(); + Ok(Arc::new(Self { + db: Data::new(args.db.clone()), + sender, + receiver: Mutex::new(receiver), + startup_netburst: config.startup_netburst, + startup_netburst_keep: config.startup_netburst_keep, + })) } #[tracing::instrument(skip_all, name = "sender")] - async fn handler(&self) -> Result<()> { + async fn worker(self: Arc) -> Result<()> { let receiver = self.receiver.lock().await; let mut futures: SendingFutures<'_> = FuturesUnordered::new(); let mut statuses: CurTransactionStatus = CurTransactionStatus::new(); @@ -77,6 +80,16 @@ impl Service { } } + fn interrupt(&self) { + if !self.sender.is_closed() { + self.sender.close(); + } + } + + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { fn handle_response( &self, response: SendingResult, futures: &mut SendingFutures<'_>, statuses: &mut CurTransactionStatus, ) { diff --git a/src/service/service.rs b/src/service/service.rs index ef60f359..3b8f4231 100644 --- a/src/service/service.rs +++ b/src/service/service.rs @@ -15,19 +15,12 @@ pub(crate) trait Service: Send + Sync { where Self: Sized; - /// Start the service. Implement the spawning of any service workers. This - /// is called after all other services have been constructed. Failure will - /// shutdown the server with an error. - async fn start(self: Arc) -> Result<()> { Ok(()) } + /// Implement the service's worker loop. The service manager spawns a + /// task and calls this function after all services have been built. + async fn worker(self: Arc) -> Result<()> { Ok(()) } - /// Stop the service. Implement the joining of any service workers and - /// cleanup of any other state. This function is asynchronous to allow that - /// gracefully, but errors cannot propagate. - async fn stop(&self) {} - - /// Interrupt the service. This may be sent prior to `stop()` as a - /// notification to improve the shutdown sequence. Implementations must be - /// robust to this being called multiple times. + /// Interrupt the service. This is sent to initiate a graceful shutdown. + /// The service worker should return from its work loop. fn interrupt(&self) {} /// Clear any caches or similar runtime state. diff --git a/src/service/services.rs b/src/service/services.rs index aeed8204..13689008 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -1,7 +1,13 @@ -use std::{collections::BTreeMap, fmt::Write, sync::Arc}; +use std::{collections::BTreeMap, fmt::Write, panic::AssertUnwindSafe, sync::Arc, time::Duration}; -use conduit::{debug, debug_info, info, trace, Result, Server}; +use conduit::{debug, debug_info, error, info, trace, utils::time, warn, Error, Result, Server}; use database::Database; +use futures_util::FutureExt; +use tokio::{ + sync::{Mutex, MutexGuard}, + task::{JoinHandle, JoinSet}, + time::sleep, +}; use crate::{ account_data, admin, appservice, globals, key_backups, media, presence, pusher, rooms, sending, @@ -24,11 +30,19 @@ pub struct Services { pub media: Arc, pub sending: Arc, + workers: Mutex, + manager: Mutex>>>, pub(crate) service: Map, pub server: Arc, pub db: Arc, } +type Workers = JoinSet; +type WorkerResult = (Arc, Result<()>); +type WorkersLocked<'a> = MutexGuard<'a, Workers>; + +const RESTART_DELAY_MS: u64 = 2500; + impl Services { pub fn build(server: Arc, db: Arc) -> Result { let mut service: Map = BTreeMap::new(); @@ -79,12 +93,74 @@ impl Services { media: build!(media::Service), sending: build!(sending::Service), globals: build!(globals::Service), + workers: Mutex::new(JoinSet::new()), + manager: Mutex::new(None), service, server, db, }) } + pub async fn start(&self) -> Result<()> { + debug_info!("Starting services..."); + + self.media.create_media_dir().await?; + globals::migrations::migrations(&self.db, &self.globals.config).await?; + globals::emerg_access::init_emergency_access(); + + let mut workers = self.workers.lock().await; + for service in self.service.values() { + self.start_worker(&mut workers, service).await?; + } + + debug!("Starting service manager..."); + let manager = async move { crate::services().manager().await }; + let manager = self.server.runtime().spawn(manager); + _ = self.manager.lock().await.insert(manager); + + if self.globals.allow_check_for_updates() { + let handle = globals::updates::start_check_for_updates_task(); + _ = self.globals.updates_handle.lock().await.insert(handle); + } + + debug_info!("Services startup complete."); + Ok(()) + } + + pub async fn stop(&self) { + info!("Shutting down services..."); + self.interrupt(); + + debug!("Waiting for update worker..."); + if let Some(updates_handle) = self.globals.updates_handle.lock().await.take() { + updates_handle.abort(); + _ = updates_handle.await; + } + + debug!("Stopping service manager..."); + if let Some(manager) = self.manager.lock().await.take() { + if let Err(e) = manager.await { + error!("Manager shutdown error: {e:?}"); + } + } + + debug_info!("Services shutdown complete."); + } + + pub async fn clear_cache(&self) { + for service in self.service.values() { + service.clear_cache(); + } + + //TODO + self.rooms + .spaces + .roomid_spacehierarchy_cache + .lock() + .await + .clear(); + } + pub async fn memory_usage(&self) -> Result { let mut out = String::new(); for service in self.service.values() { @@ -104,65 +180,97 @@ impl Services { Ok(out) } - pub async fn clear_cache(&self) { - for service in self.service.values() { - service.clear_cache(); - } - - //TODO - self.rooms - .spaces - .roomid_spacehierarchy_cache - .lock() - .await - .clear(); - } - - pub async fn start(&self) -> Result<()> { - debug_info!("Starting services"); - - self.media.create_media_dir().await?; - globals::migrations::migrations(&self.db, &self.globals.config).await?; - globals::emerg_access::init_emergency_access(); - - for (name, service) in &self.service { - debug!("Starting {name}"); - service.clone().start().await?; - } - - if self.globals.allow_check_for_updates() { - let handle = globals::updates::start_check_for_updates_task(); - _ = self.globals.updates_handle.lock().await.insert(handle); - } - - debug_info!("Services startup complete."); - Ok(()) - } - - pub fn interrupt(&self) { + fn interrupt(&self) { debug!("Interrupting services..."); - for (name, service) in &self.service { trace!("Interrupting {name}"); service.interrupt(); } } - pub async fn stop(&self) { - info!("Shutting down services"); - self.interrupt(); - - debug!("Waiting for update worker..."); - if let Some(updates_handle) = self.globals.updates_handle.lock().await.take() { - updates_handle.abort(); - _ = updates_handle.await; + async fn manager(&self) -> Result<()> { + loop { + let mut workers = self.workers.lock().await; + tokio::select! { + result = workers.join_next() => match result { + Some(Ok(result)) => self.handle_result(&mut workers, result).await?, + Some(Err(error)) => self.handle_abort(&mut workers, Error::from(error)).await?, + None => break, + } + } } - for (name, service) in &self.service { - debug!("Waiting for {name} ..."); - service.stop().await; + debug!("Worker manager finished"); + Ok(()) + } + + async fn handle_abort(&self, _workers: &mut WorkersLocked<'_>, error: Error) -> Result<()> { + // not supported until service can be associated with abort + unimplemented!("unexpected worker task abort {error:?}"); + } + + async fn handle_result(&self, workers: &mut WorkersLocked<'_>, result: WorkerResult) -> Result<()> { + let (service, result) = result; + match result { + Ok(()) => self.handle_finished(workers, &service).await, + Err(error) => self.handle_error(workers, &service, error).await, + } + } + + async fn handle_finished(&self, _workers: &mut WorkersLocked<'_>, service: &Arc) -> Result<()> { + debug!("service {:?} worker finished", service.name()); + Ok(()) + } + + async fn handle_error( + &self, workers: &mut WorkersLocked<'_>, service: &Arc, error: Error, + ) -> Result<()> { + let name = service.name(); + error!("service {name:?} worker error: {error}"); + + if !error.is_panic() { + return Ok(()); } - debug_info!("Services shutdown complete."); + if !self.server.running() { + return Ok(()); + } + + let delay = Duration::from_millis(RESTART_DELAY_MS); + warn!("service {name:?} worker restarting after {} delay", time::pretty(delay)); + sleep(delay).await; + + self.start_worker(workers, service).await + } + + /// Start the worker in a task for the service. + async fn start_worker(&self, workers: &mut WorkersLocked<'_>, service: &Arc) -> Result<()> { + if !self.server.running() { + return Err(Error::Err(format!( + "Service {:?} worker not starting during server shutdown.", + service.name() + ))); + } + + debug!("Service {:?} worker starting...", service.name()); + workers.spawn_on(worker(service.clone()), self.server.runtime()); + + Ok(()) } } + +/// Base frame for service worker. This runs in a tokio::task. All errors and +/// panics from the worker are caught and returned cleanly. The JoinHandle +/// should never error with a panic, and if so it should propagate, but it may +/// error with an Abort which the manager should handle along with results to +/// determine if the worker should be restarted. +async fn worker(service: Arc) -> WorkerResult { + let service_ = Arc::clone(&service); + let result = AssertUnwindSafe(service_.worker()) + .catch_unwind() + .await + .map_err(Error::from_panic); + + // flattens JoinError for panic into worker's Error + (service, result.unwrap_or_else(Err)) +} From 93ec4e579bd9fc3c26216e659a673f9ff64d67bf Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 12 Jul 2024 07:37:46 +0000 Subject: [PATCH 108/158] error macro suite Signed-off-by: Jason Volk --- src/core/error.rs | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/core/error.rs b/src/core/error.rs index 8567f303..a9f44dc9 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -15,6 +15,48 @@ use ruma::{ use crate::{debug::panic_str, debug_error, error}; +#[macro_export] +macro_rules! err { + (error!($($args:tt),+)) => {{ + error!($($args),+); + $crate::error::Error::Err(format!($($args),+)) + }}; + + (debug_error!($($args:tt),+)) => {{ + debug_error!($($args),+); + $crate::error::Error::Err(format!($($args),+)) + }}; + + ($variant:ident(error!($($args:tt),+))) => {{ + error!($($args),+); + $crate::error::Error::$variant(format!($($args),+)) + }}; + + ($variant:ident(debug_error!($($args:tt),+))) => {{ + debug_error!($($args),+); + $crate::error::Error::$variant(format!($($args),+)) + }}; + + ($variant:ident(format!($($args:tt),+))) => { + $crate::error::Error::$variant(format!($($args),+)) + }; + + ($variant:ident($($args:tt),+)) => { + $crate::error::Error::$variant(format!($($args),+)) + }; + + ($string:literal$(,)? $($args:tt),*) => { + $crate::error::Error::Err(format!($string, $($args),*)) + }; +} + +#[macro_export] +macro_rules! Err { + ($($args:tt)*) => { + Err($crate::err!($($args)*)) + }; +} + #[derive(thiserror::Error)] pub enum Error { #[error("PANIC!")] From 4cc92dd17571329bd138f2be0552db26b7026055 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 12 Jul 2024 07:41:01 +0000 Subject: [PATCH 109/158] refactor Error::bad_config Signed-off-by: Jason Volk --- src/api/router/auth.rs | 3 +- src/api/routes.rs | 4 +-- src/core/config/check.rs | 52 ++++++++++++++++++++------------- src/core/config/mod.rs | 6 ++-- src/core/error.rs | 36 ++++++++++------------- src/main/tracing.rs | 19 ++++++------ src/service/globals/resolver.rs | 7 ++--- src/service/rooms/alias/mod.rs | 4 +-- src/service/sending/resolve.rs | 3 +- src/service/sending/send.rs | 3 +- 10 files changed, 71 insertions(+), 66 deletions(-) diff --git a/src/api/router/auth.rs b/src/api/router/auth.rs index 08a08e08..6c2922b9 100644 --- a/src/api/router/auth.rs +++ b/src/api/router/auth.rs @@ -6,6 +6,7 @@ use axum_extra::{ typed_header::TypedHeaderRejectionReason, TypedHeader, }; +use conduit::Err; use http::uri::PathAndQuery; use ruma::{ api::{client::error::ErrorKind, AuthScheme, Metadata}, @@ -183,7 +184,7 @@ fn auth_appservice(request: &Request, info: Box) -> Result) -> Result { if !services().globals.allow_federation() { - return Err(Error::bad_config("Federation is disabled.")); + return Err!(Config("allow_federation", "Federation is disabled.")); } let TypedHeader(Authorization(x_matrix)) = request diff --git a/src/api/routes.rs b/src/api/routes.rs index b22a32cb..3a8b2c74 100644 --- a/src/api/routes.rs +++ b/src/api/routes.rs @@ -3,7 +3,7 @@ use axum::{ routing::{any, get, post}, Router, }; -use conduit::{Error, Server}; +use conduit::{err, Error, Server}; use http::Uri; use ruma::api::client::error::ErrorKind; @@ -236,4 +236,4 @@ async fn initial_sync(_uri: Uri) -> impl IntoResponse { Error::BadRequest(ErrorKind::GuestAccessForbidden, "Guest access not implemented") } -async fn federation_disabled() -> impl IntoResponse { Error::bad_config("Federation is disabled.") } +async fn federation_disabled() -> impl IntoResponse { err!(Config("allow_federation", "Federation is disabled.")) } diff --git a/src/core/config/check.rs b/src/core/config/check.rs index 95b4df4e..67d19c72 100644 --- a/src/core/config/check.rs +++ b/src/core/config/check.rs @@ -1,6 +1,9 @@ -use crate::{error::Error, warn, Config}; +use crate::{error, error::Error, info, warn, Config, Err}; pub fn check(config: &Config) -> Result<(), Error> { + #[cfg(debug_assertions)] + info!("Note: conduwuit was built without optimisations (i.e. debug build)"); + #[cfg(all(feature = "rocksdb", not(feature = "sha256_media")))] // prevents catching this in `--all-features` warn!( "Note the rocksdb feature was deleted from conduwuit. SQLite support was removed and RocksDB is the only \ @@ -16,7 +19,7 @@ pub fn check(config: &Config) -> Result<(), Error> { config.warn_unknown_key(); if config.sentry && config.sentry_endpoint.is_none() { - return Err(Error::bad_config("Sentry cannot be enabled without an endpoint set")); + return Err!(Config("sentry_endpoint", "Sentry cannot be enabled without an endpoint set")); } #[cfg(all(feature = "hardened_malloc", feature = "jemalloc"))] @@ -27,7 +30,8 @@ pub fn check(config: &Config) -> Result<(), Error> { #[cfg(not(unix))] if config.unix_socket_path.is_some() { - return Err(Error::bad_config( + return Err!(Config( + "unix_socket_path", "UNIX socket support is only available on *nix platforms. Please remove \"unix_socket_path\" from your \ config.", )); @@ -44,7 +48,7 @@ pub fn check(config: &Config) -> Result<(), Error> { if Path::new("/proc/vz").exists() /* Guest */ && !Path::new("/proc/bz").exists() /* Host */ { - crate::error!( + error!( "You are detected using OpenVZ with a loopback/localhost listening address of {addr}. If you \ are using OpenVZ for containers and you use NAT-based networking to communicate with the \ host and guest, this will NOT work. Please change this to \"0.0.0.0\". If this is expected, \ @@ -53,7 +57,7 @@ pub fn check(config: &Config) -> Result<(), Error> { } if Path::new("/.dockerenv").exists() { - crate::error!( + error!( "You are detected using Docker with a loopback/localhost listening address of {addr}. If you \ are using a reverse proxy on the host and require communication to conduwuit in the Docker \ container via NAT-based networking, this will NOT work. Please change this to \"0.0.0.0\". \ @@ -62,7 +66,7 @@ pub fn check(config: &Config) -> Result<(), Error> { } if Path::new("/run/.containerenv").exists() { - crate::error!( + error!( "You are detected using Podman with a loopback/localhost listening address of {addr}. If you \ are using a reverse proxy on the host and require communication to conduwuit in the Podman \ container via NAT-based networking, this will NOT work. Please change this to \"0.0.0.0\". \ @@ -75,36 +79,40 @@ pub fn check(config: &Config) -> Result<(), Error> { // rocksdb does not allow max_log_files to be 0 if config.rocksdb_max_log_files == 0 { - return Err(Error::bad_config( - "rocksdb_max_log_files cannot be 0. Please set a value at least 1.", + return Err!(Config( + "max_log_files", + "rocksdb_max_log_files cannot be 0. Please set a value at least 1." )); } // yeah, unless the user built a debug build hopefully for local testing only #[cfg(not(debug_assertions))] if config.server_name == "your.server.name" { - return Err(Error::bad_config( - "You must specify a valid server name for production usage of conduwuit.", + return Err!(Config( + "server_name", + "You must specify a valid server name for production usage of conduwuit." )); } - #[cfg(debug_assertions)] - crate::info!("Note: conduwuit was built without optimisations (i.e. debug build)"); - // check if the user specified a registration token as `""` if config.registration_token == Some(String::new()) { - return Err(Error::bad_config("Registration token was specified but is empty (\"\")")); + return Err!(Config( + "registration_token", + "Registration token was specified but is empty (\"\")" + )); } if config.max_request_size < 5_120_000 { - return Err(Error::bad_config("Max request size is less than 5MB. Please increase it.")); + return Err!(Config( + "max_request_size", + "Max request size is less than 5MB. Please increase it." + )); } // check if user specified valid IP CIDR ranges on startup for cidr in &config.ip_range_denylist { if let Err(e) = ipaddress::IPAddress::parse(cidr) { - crate::error!("Error parsing specified IP CIDR range from string: {e}"); - return Err(Error::bad_config("Error parsing specified IP CIDR ranges from strings")); + return Err!(Config("ip_range_denylist", "Parsing specified IP CIDR range from string: {e}.")); } } @@ -112,13 +120,14 @@ pub fn check(config: &Config) -> Result<(), Error> { && !config.yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse && config.registration_token.is_none() { - return Err(Error::bad_config( + return Err!(Config( + "registration_token", "!! You have `allow_registration` enabled without a token configured in your config which means you are \ allowing ANYONE to register on your conduwuit instance without any 2nd-step (e.g. registration token).\n If this is not the intended behaviour, please set a registration token with the `registration_token` config option.\n For security and safety reasons, conduwuit will shut down. If you are extra sure this is the desired behaviour you \ want, please set the following config option to true: -`yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse`", +`yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse`" )); } @@ -135,8 +144,9 @@ For security and safety reasons, conduwuit will shut down. If you are extra sure } if config.allow_outgoing_presence && !config.allow_local_presence { - return Err(Error::bad_config( - "Outgoing presence requires allowing local presence. Please enable \"allow_local_presence\".", + return Err!(Config( + "allow_local_presence", + "Outgoing presence requires allowing local presence. Please enable 'allow_local_presence'." )); } diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index aadfe564..d12e7780 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -24,7 +24,7 @@ use url::Url; pub use self::check::check; use self::proxy::ProxyConfig; -use crate::error::Error; +use crate::{error::Error, Err}; pub mod check; pub mod proxy; @@ -433,13 +433,13 @@ impl Config { }; let config = match raw_config.extract::() { - Err(e) => return Err(Error::BadConfig(format!("{e}"))), + Err(e) => return Err!("There was a problem with your configuration file: {e}"), Ok(config) => config, }; // don't start if we're listening on both UNIX sockets and TCP at same time if Self::is_dual_listening(&raw_config) { - return Err(Error::bad_config("dual listening on UNIX and TCP sockets not allowed.")); + return Err!(Config("address", "dual listening on UNIX and TCP sockets not allowed.")); }; Ok(config) diff --git a/src/core/error.rs b/src/core/error.rs index a9f44dc9..77faae3a 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -18,35 +18,36 @@ use crate::{debug::panic_str, debug_error, error}; #[macro_export] macro_rules! err { (error!($($args:tt),+)) => {{ - error!($($args),+); - $crate::error::Error::Err(format!($($args),+)) + $crate::error!($($args),+); + $crate::error::Error::Err(std::format!($($args),+)) }}; (debug_error!($($args:tt),+)) => {{ - debug_error!($($args),+); - $crate::error::Error::Err(format!($($args),+)) + $crate::debug_error!($($args),+); + $crate::error::Error::Err(std::format!($($args),+)) }}; ($variant:ident(error!($($args:tt),+))) => {{ - error!($($args),+); - $crate::error::Error::$variant(format!($($args),+)) + $crate::error!($($args),+); + $crate::error::Error::$variant(std::format!($($args),+)) }}; ($variant:ident(debug_error!($($args:tt),+))) => {{ - debug_error!($($args),+); - $crate::error::Error::$variant(format!($($args),+)) + $crate::debug_error!($($args),+); + $crate::error::Error::$variant(std::format!($($args),+)) }}; - ($variant:ident(format!($($args:tt),+))) => { - $crate::error::Error::$variant(format!($($args),+)) - }; + (Config($item:literal, $($args:tt),+)) => {{ + $crate::error!(config = %$item, $($args),+); + $crate::error::Error::Config($item, std::format!($($args),+)) + }}; ($variant:ident($($args:tt),+)) => { - $crate::error::Error::$variant(format!($($args),+)) + $crate::error::Error::$variant(std::format!($($args),+)) }; ($string:literal$(,)? $($args:tt),*) => { - $crate::error::Error::Err(format!($string, $($args),*)) + $crate::error::Error::Err(std::format!($string, $($args),*)) }; } @@ -123,8 +124,8 @@ pub enum Error { // conduwuit #[error("Arithmetic operation failed: {0}")] Arithmetic(&'static str), - #[error("There was a problem with your configuration: {0}")] - BadConfig(String), + #[error("There was a problem with the '{0}' directive in your configuration: {1}")] + Config(&'static str, String), #[error("{0}")] BadDatabase(&'static str), #[error("{0}")] @@ -145,11 +146,6 @@ impl Error { Self::BadDatabase(message) } - pub fn bad_config(message: &str) -> Self { - error!("BadConfig: {}", message); - Self::BadConfig(message.to_owned()) - } - #[must_use] pub fn from_panic(e: Box) -> Self { Self::Panic(panic_str(&e), e) } diff --git a/src/main/tracing.rs b/src/main/tracing.rs index 67977088..0217f38a 100644 --- a/src/main/tracing.rs +++ b/src/main/tracing.rs @@ -2,9 +2,9 @@ use std::sync::Arc; use conduit::{ config::Config, - debug_warn, + debug_warn, err, log::{capture, LogLevelReloadHandles}, - Error, Result, + Result, }; use tracing_subscriber::{layer::SubscriberExt, reload, EnvFilter, Layer, Registry}; @@ -17,8 +17,7 @@ pub(crate) type TracingFlameGuard = (); pub(crate) fn init(config: &Config) -> Result<(LogLevelReloadHandles, TracingFlameGuard, Arc)> { let reload_handles = LogLevelReloadHandles::default(); - let console_filter = - EnvFilter::try_new(&config.log).map_err(|e| Error::BadConfig(format!("in the 'log' setting: {e}.")))?; + let console_filter = EnvFilter::try_new(&config.log).map_err(|e| err!(Config("log", "{e}.")))?; let console_layer = tracing_subscriber::fmt::Layer::new(); let (console_reload_filter, console_reload_handle) = reload::Layer::new(console_filter.clone()); reload_handles.add("console", Box::new(console_reload_handle)); @@ -32,8 +31,8 @@ pub(crate) fn init(config: &Config) -> Result<(LogLevelReloadHandles, TracingFla #[cfg(feature = "sentry_telemetry")] let subscriber = { - let sentry_filter = EnvFilter::try_new(&config.sentry_filter) - .map_err(|e| Error::BadConfig(format!("in the 'sentry_filter' setting: {e}.")))?; + let sentry_filter = + EnvFilter::try_new(&config.sentry_filter).map_err(|e| err!(Config("sentry_filter", "{e}.")))?; let sentry_layer = sentry_tracing::layer(); let (sentry_reload_filter, sentry_reload_handle) = reload::Layer::new(sentry_filter); reload_handles.add("sentry", Box::new(sentry_reload_handle)); @@ -44,9 +43,9 @@ pub(crate) fn init(config: &Config) -> Result<(LogLevelReloadHandles, TracingFla let (subscriber, flame_guard) = { let (flame_layer, flame_guard) = if config.tracing_flame { let flame_filter = EnvFilter::try_new(&config.tracing_flame_filter) - .map_err(|e| Error::BadConfig(format!("in the 'tracing_flame_filter' setting: {e}.")))?; + .map_err(|e| err!(Config("tracing_flame_filter", "{e}.")))?; let (flame_layer, flame_guard) = tracing_flame::FlameLayer::with_file(&config.tracing_flame_output_path) - .map_err(|e| Error::BadConfig(format!("in the 'tracing_flame_output_path' setting: {e}.")))?; + .map_err(|e| err!(Config("tracing_flame_output_path", "{e}.")))?; let flame_layer = flame_layer .with_empty_samples(false) .with_filter(flame_filter); @@ -55,8 +54,8 @@ pub(crate) fn init(config: &Config) -> Result<(LogLevelReloadHandles, TracingFla (None, None) }; - let jaeger_filter = EnvFilter::try_new(&config.jaeger_filter) - .map_err(|e| Error::BadConfig(format!("in the 'jaeger_filter' setting: {e}.")))?; + let jaeger_filter = + EnvFilter::try_new(&config.jaeger_filter).map_err(|e| err!(Config("jaeger_filter", "{e}.")))?; let jaeger_layer = config.allow_jaeger.then(|| { opentelemetry::global::set_text_map_propagator(opentelemetry_jaeger::Propagator::new()); let tracer = opentelemetry_jaeger::new_agent_pipeline() diff --git a/src/service/globals/resolver.rs b/src/service/globals/resolver.rs index 3082f2fd..3002decf 100644 --- a/src/service/globals/resolver.rs +++ b/src/service/globals/resolver.rs @@ -7,7 +7,7 @@ use std::{ time::Duration, }; -use conduit::{error, Config, Error, Result}; +use conduit::{error, Config, Result}; use hickory_resolver::TokioAsyncResolver; use reqwest::dns::{Addrs, Name, Resolve, Resolving}; use ruma::OwnedServerName; @@ -33,10 +33,7 @@ impl Resolver { #[allow(clippy::as_conversions, clippy::cast_sign_loss, clippy::cast_possible_truncation)] pub(super) fn new(config: &Config) -> Self { let (sys_conf, mut opts) = hickory_resolver::system_conf::read_system_conf() - .map_err(|e| { - error!("Failed to set up hickory dns resolver with system config: {e}"); - Error::bad_config("Failed to set up hickory dns resolver with system config.") - }) + .inspect_err(|e| error!("Failed to set up hickory dns resolver with system config: {e}")) .expect("DNS system config must be valid"); let mut conf = hickory_resolver::config::ResolverConfig::new(); diff --git a/src/service/rooms/alias/mod.rs b/src/service/rooms/alias/mod.rs index 4af1035e..97eb0f48 100644 --- a/src/service/rooms/alias/mod.rs +++ b/src/service/rooms/alias/mod.rs @@ -3,7 +3,7 @@ mod remote; use std::sync::Arc; -use conduit::{Error, Result}; +use conduit::{err, Error, Result}; use data::Data; use ruma::{ api::{appservice, client::error::ErrorKind}, @@ -171,7 +171,7 @@ impl Service { .rooms .alias .resolve_local_alias(room_alias)? - .ok_or_else(|| Error::bad_config("Room does not exist."))?, + .ok_or_else(|| err!(BadConfig("Room does not exist.")))?, )); } } diff --git a/src/service/sending/resolve.rs b/src/service/sending/resolve.rs index d38509ba..91041a2f 100644 --- a/src/service/sending/resolve.rs +++ b/src/service/sending/resolve.rs @@ -5,6 +5,7 @@ use std::{ time::SystemTime, }; +use conduit::Err; use hickory_resolver::{error::ResolveError, lookup::SrvLookup}; use ipaddress::IPAddress; use ruma::{OwnedServerName, ServerName}; @@ -354,7 +355,7 @@ fn handle_resolve_error(e: &ResolveError) -> Result<()> { fn validate_dest(dest: &ServerName) -> Result<()> { if dest == services().globals.server_name() { - return Err(Error::bad_config("Won't send federation request to ourselves")); + return Err!("Won't send federation request to ourselves"); } if dest.is_ip_literal() || IPAddress::is_valid(dest.host()) { diff --git a/src/service/sending/send.rs b/src/service/sending/send.rs index f4825ff3..18a98828 100644 --- a/src/service/sending/send.rs +++ b/src/service/sending/send.rs @@ -1,5 +1,6 @@ use std::{fmt::Debug, mem}; +use conduit::Err; use http::{header::AUTHORIZATION, HeaderValue}; use ipaddress::IPAddress; use reqwest::{Client, Method, Request, Response, Url}; @@ -26,7 +27,7 @@ where T: OutgoingRequest + Debug + Send, { if !services().globals.allow_federation() { - return Err(Error::bad_config("Federation is disabled.")); + return Err!(Config("allow_federation", "Federation is disabled.")); } let actual = resolve::get_actual_dest(dest).await?; From 4600c7f32de2683c465d3dcfaf4ddac8a7c250c6 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 12 Jul 2024 20:13:55 +0000 Subject: [PATCH 110/158] move infallible handling into error Signed-off-by: Jason Volk --- src/core/error.rs | 12 +++++++++++- src/core/utils/mod.rs | 13 ------------- src/router/serve/unix.rs | 12 ++++++++---- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/core/error.rs b/src/core/error.rs index 77faae3a..01460217 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -196,7 +196,11 @@ impl UnwindSafe for Error {} impl RefUnwindSafe for Error {} impl From for Error { - fn from(i: Infallible) -> Self { match i {} } + #[cold] + #[inline(never)] + fn from(_i: Infallible) -> Self { + panic!("infallible error should never exist"); + } } impl fmt::Debug for Error { @@ -273,6 +277,12 @@ pub fn inspect_debug_log(error: &E) { debug_error!("{error:?}"); } +#[cold] +#[inline(never)] +pub fn infallible(_e: &Infallible) { + panic!("infallible error should never exist"); +} + impl axum::response::IntoResponse for Error { fn into_response(self) -> axum::response::Response { let response: UiaaResponse = self.into(); diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 4e5ba480..bbd52829 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -26,21 +26,8 @@ pub use string::{str_from_bytes, string_from_bytes}; pub use sys::available_parallelism; pub use time::now_millis as millis_since_unix_epoch; -use crate::Result; - pub fn clamp(val: T, min: T, max: T) -> T { cmp::min(cmp::max(val, min), max) } -/// Boilerplate for wraps which are typed to never error. -/// -/// * -#[must_use] -pub fn unwrap_infallible(result: Result) -> T { - match result { - Ok(val) => val, - Err(err) => match err {}, - } -} - #[must_use] pub fn generate_keypair() -> Vec { let mut value = rand::string(8).as_bytes().to_vec(); diff --git a/src/router/serve/unix.rs b/src/router/serve/unix.rs index 8373b749..e876b2ac 100644 --- a/src/router/serve/unix.rs +++ b/src/router/serve/unix.rs @@ -10,7 +10,7 @@ use axum::{ extract::{connect_info::IntoMakeServiceWithConnectInfo, Request}, Router, }; -use conduit::{debug_error, trace, utils, Error, Result, Server}; +use conduit::{debug_error, error::infallible, trace, Error, Result, Server}; use hyper::{body::Incoming, service::service_fn}; use hyper_util::{ rt::{TokioExecutor, TokioIo}, @@ -24,7 +24,6 @@ use tokio::{ }; use tower::{Service, ServiceExt}; use tracing::{debug, info, warn}; -use utils::unwrap_infallible; type MakeService = IntoMakeServiceWithConnectInfo; @@ -62,9 +61,14 @@ async fn accept( let socket = TokioIo::new(socket); trace!(?listener, ?socket, ?remote, "accepted"); - let called = unwrap_infallible(app.call(NULL_ADDR).await); - let handler = service_fn(move |req: Request| called.clone().oneshot(req)); + let called = app + .call(NULL_ADDR) + .await + .inspect_err(infallible) + .expect("infallible"); + let service = move |req: Request| called.clone().oneshot(req); + let handler = service_fn(service); let task = async move { // bug on darwin causes all results to be errors. do not unwrap this _ = builder.serve_connection(socket, handler).await; From 4a6f089b23fccb5279a200546e81937199ad56f3 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 13 Jul 2024 01:24:37 +0000 Subject: [PATCH 111/158] move some config checks into check unit Signed-off-by: Jason Volk --- src/core/config/check.rs | 61 ++++++++++++++++++++++++++-- src/core/config/mod.rs | 88 ++++++++-------------------------------- 2 files changed, 74 insertions(+), 75 deletions(-) diff --git a/src/core/config/check.rs b/src/core/config/check.rs index 67d19c72..20a61b70 100644 --- a/src/core/config/check.rs +++ b/src/core/config/check.rs @@ -1,6 +1,10 @@ -use crate::{error, error::Error, info, warn, Config, Err}; +use figment::Figment; -pub fn check(config: &Config) -> Result<(), Error> { +use super::DEPRECATED_KEYS; +use crate::{debug, error, info, warn, Config, Err, Result}; + +#[allow(clippy::cognitive_complexity)] +pub fn check(config: &Config) -> Result<()> { #[cfg(debug_assertions)] info!("Note: conduwuit was built without optimisations (i.e. debug build)"); @@ -15,8 +19,8 @@ pub fn check(config: &Config) -> Result<(), Error> { forwards-compatible way. Please update your build script to remove this feature." ); - config.warn_deprecated(); - config.warn_unknown_key(); + warn_deprecated(config); + warn_unknown_key(config); if config.sentry && config.sentry_endpoint.is_none() { return Err!(Config("sentry_endpoint", "Sentry cannot be enabled without an endpoint set")); @@ -183,3 +187,52 @@ For security and safety reasons, conduwuit will shut down. If you are extra sure Ok(()) } + +/// Iterates over all the keys in the config file and warns if there is a +/// deprecated key specified +fn warn_deprecated(config: &Config) { + debug!("Checking for deprecated config keys"); + let mut was_deprecated = false; + for key in config + .catchall + .keys() + .filter(|key| DEPRECATED_KEYS.iter().any(|s| s == key)) + { + warn!("Config parameter \"{}\" is deprecated, ignoring.", key); + was_deprecated = true; + } + + if was_deprecated { + warn!( + "Read conduwuit config documentation at https://conduwuit.puppyirl.gay/configuration.html and check your \ + configuration if any new configuration parameters should be adjusted" + ); + } +} + +/// iterates over all the catchall keys (unknown config options) and warns +/// if there are any. +fn warn_unknown_key(config: &Config) { + debug!("Checking for unknown config keys"); + for key in config + .catchall + .keys() + .filter(|key| "config".to_owned().ne(key.to_owned()) /* "config" is expected */) + { + warn!("Config parameter \"{}\" is unknown to conduwuit, ignoring.", key); + } +} + +/// Checks the presence of the `address` and `unix_socket_path` keys in the +/// raw_config, exiting the process if both keys were detected. +pub(super) fn is_dual_listening(raw_config: &Figment) -> Result<()> { + let contains_address = raw_config.contains("address"); + let contains_unix_socket = raw_config.contains("unix_socket_path"); + if contains_address && contains_unix_socket { + return Err!( + "TOML keys \"address\" and \"unix_socket_path\" were both defined. Please specify only one option." + ); + } + + Ok(()) +} diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index d12e7780..16c7b1a8 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -19,30 +19,15 @@ use ruma::{ api::client::discovery::discover_support::ContactRole, OwnedRoomId, OwnedServerName, OwnedUserId, RoomVersionId, }; use serde::{de::IgnoredAny, Deserialize}; -use tracing::{debug, error, warn}; use url::Url; pub use self::check::check; use self::proxy::ProxyConfig; -use crate::{error::Error, Err}; +use crate::{error::Error, Err, Result}; pub mod check; pub mod proxy; -#[derive(Deserialize, Clone, Debug)] -#[serde(transparent)] -struct ListeningPort { - #[serde(with = "either::serde_untagged")] - ports: Either>, -} - -#[derive(Deserialize, Clone, Debug)] -#[serde(transparent)] -struct ListeningAddr { - #[serde(with = "either::serde_untagged")] - addrs: Either>, -} - /// all the config options for conduwuit #[derive(Clone, Debug, Deserialize)] #[allow(clippy::struct_excessive_bools)] @@ -397,6 +382,20 @@ pub struct WellKnownConfig { pub support_mxid: Option, } +#[derive(Deserialize, Clone, Debug)] +#[serde(transparent)] +struct ListeningPort { + #[serde(with = "either::serde_untagged")] + ports: Either>, +} + +#[derive(Deserialize, Clone, Debug)] +#[serde(transparent)] +struct ListeningAddr { + #[serde(with = "either::serde_untagged")] + addrs: Either>, +} + const DEPRECATED_KEYS: &[&str] = &[ "cache_capacity", "max_concurrent_requests", @@ -410,7 +409,7 @@ const DEPRECATED_KEYS: &[&str] = &[ impl Config { /// Initialize config - pub fn new(path: Option) -> Result { + pub fn new(path: Option) -> Result { let raw_config = if let Some(config_file_env) = Env::var("CONDUIT_CONFIG") { Figment::new() .merge(Toml::file(config_file_env).nested()) @@ -438,64 +437,11 @@ impl Config { }; // don't start if we're listening on both UNIX sockets and TCP at same time - if Self::is_dual_listening(&raw_config) { - return Err!(Config("address", "dual listening on UNIX and TCP sockets not allowed.")); - }; + check::is_dual_listening(&raw_config)?; Ok(config) } - /// Iterates over all the keys in the config file and warns if there is a - /// deprecated key specified - pub(crate) fn warn_deprecated(&self) { - debug!("Checking for deprecated config keys"); - let mut was_deprecated = false; - for key in self - .catchall - .keys() - .filter(|key| DEPRECATED_KEYS.iter().any(|s| s == key)) - { - warn!("Config parameter \"{}\" is deprecated, ignoring.", key); - was_deprecated = true; - } - - if was_deprecated { - warn!( - "Read conduwuit config documentation at https://conduwuit.puppyirl.gay/configuration.html and check \ - your configuration if any new configuration parameters should be adjusted" - ); - } - } - - /// iterates over all the catchall keys (unknown config options) and warns - /// if there are any. - pub(crate) fn warn_unknown_key(&self) { - debug!("Checking for unknown config keys"); - for key in self - .catchall - .keys() - .filter(|key| "config".to_owned().ne(key.to_owned()) /* "config" is expected */) - { - warn!("Config parameter \"{}\" is unknown to conduwuit, ignoring.", key); - } - } - - /// Checks the presence of the `address` and `unix_socket_path` keys in the - /// raw_config, exiting the process if both keys were detected. - fn is_dual_listening(raw_config: &Figment) -> bool { - let check_address = raw_config.find_value("address"); - let check_unix_socket = raw_config.find_value("unix_socket_path"); - - // are the check_address and check_unix_socket keys both Ok (specified) at the - // same time? - if check_address.is_ok() && check_unix_socket.is_ok() { - error!("TOML keys \"address\" and \"unix_socket_path\" were both defined. Please specify only one option."); - return true; - } - - false - } - #[must_use] pub fn get_bind_addrs(&self) -> Vec { let mut addrs = Vec::new(); From b87f1649d4e1f2e212188e93c58d8d52fbf1ad84 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 13 Jul 2024 02:15:15 +0000 Subject: [PATCH 112/158] split error into directory Signed-off-by: Jason Volk --- src/api/client/membership.rs | 10 +- src/core/error.rs | 384 ----------------------------------- src/core/error/err.rs | 42 ++++ src/core/error/log.rs | 74 +++++++ src/core/error/mod.rs | 156 ++++++++++++++ src/core/error/panic.rs | 41 ++++ src/core/error/response.rs | 88 ++++++++ 7 files changed, 404 insertions(+), 391 deletions(-) delete mode 100644 src/core/error.rs create mode 100644 src/core/error/err.rs create mode 100644 src/core/error/log.rs create mode 100644 src/core/error/mod.rs create mode 100644 src/core/error/panic.rs create mode 100644 src/core/error/response.rs diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 1b70fdec..c25c08d1 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -1273,16 +1273,12 @@ async fn make_join_request( make_join_counter = make_join_counter.saturating_add(1); if let Err(ref e) = make_join_response { - trace!("make_join ErrorKind string: {:?}", e.error_code().to_string()); + trace!("make_join ErrorKind string: {:?}", e.kind().to_string()); // converting to a string is necessary (i think) because ruma is forcing us to // fill in the struct for M_INCOMPATIBLE_ROOM_VERSION - if e.error_code() - .to_string() - .contains("M_INCOMPATIBLE_ROOM_VERSION") - || e.error_code() - .to_string() - .contains("M_UNSUPPORTED_ROOM_VERSION") + if e.kind().to_string().contains("M_INCOMPATIBLE_ROOM_VERSION") + || e.kind().to_string().contains("M_UNSUPPORTED_ROOM_VERSION") { incompatible_room_version_count = incompatible_room_version_count.saturating_add(1); } diff --git a/src/core/error.rs b/src/core/error.rs deleted file mode 100644 index 01460217..00000000 --- a/src/core/error.rs +++ /dev/null @@ -1,384 +0,0 @@ -use std::{ - any::Any, - convert::Infallible, - fmt, - panic::{RefUnwindSafe, UnwindSafe}, -}; - -use bytes::BytesMut; -use http::StatusCode; -use http_body_util::Full; -use ruma::{ - api::{client::uiaa::UiaaResponse, OutgoingResponse}, - OwnedServerName, -}; - -use crate::{debug::panic_str, debug_error, error}; - -#[macro_export] -macro_rules! err { - (error!($($args:tt),+)) => {{ - $crate::error!($($args),+); - $crate::error::Error::Err(std::format!($($args),+)) - }}; - - (debug_error!($($args:tt),+)) => {{ - $crate::debug_error!($($args),+); - $crate::error::Error::Err(std::format!($($args),+)) - }}; - - ($variant:ident(error!($($args:tt),+))) => {{ - $crate::error!($($args),+); - $crate::error::Error::$variant(std::format!($($args),+)) - }}; - - ($variant:ident(debug_error!($($args:tt),+))) => {{ - $crate::debug_error!($($args),+); - $crate::error::Error::$variant(std::format!($($args),+)) - }}; - - (Config($item:literal, $($args:tt),+)) => {{ - $crate::error!(config = %$item, $($args),+); - $crate::error::Error::Config($item, std::format!($($args),+)) - }}; - - ($variant:ident($($args:tt),+)) => { - $crate::error::Error::$variant(std::format!($($args),+)) - }; - - ($string:literal$(,)? $($args:tt),*) => { - $crate::error::Error::Err(std::format!($string, $($args),*)) - }; -} - -#[macro_export] -macro_rules! Err { - ($($args:tt)*) => { - Err($crate::err!($($args)*)) - }; -} - -#[derive(thiserror::Error)] -pub enum Error { - #[error("PANIC!")] - PanicAny(Box), - #[error("PANIC! {0}")] - Panic(&'static str, Box), - - // std - #[error("{0}")] - Fmt(#[from] fmt::Error), - #[error("I/O error: {0}")] - Io(#[from] std::io::Error), - #[error("{0}")] - Utf8Error(#[from] std::str::Utf8Error), - #[error("{0}")] - FromUtf8Error(#[from] std::string::FromUtf8Error), - #[error("{0}")] - TryFromSliceError(#[from] std::array::TryFromSliceError), - #[error("{0}")] - TryFromIntError(#[from] std::num::TryFromIntError), - #[error("{0}")] - ParseIntError(#[from] std::num::ParseIntError), - #[error("{0}")] - ParseFloatError(#[from] std::num::ParseFloatError), - - // third-party - #[error("Join error: {0}")] - JoinError(#[from] tokio::task::JoinError), - #[error("Regex error: {0}")] - Regex(#[from] regex::Error), - #[error("Tracing filter error: {0}")] - TracingFilter(#[from] tracing_subscriber::filter::ParseError), - #[error("Tracing reload error: {0}")] - TracingReload(#[from] tracing_subscriber::reload::Error), - #[error("Image error: {0}")] - Image(#[from] image::error::ImageError), - #[error("Request error: {0}")] - Reqwest(#[from] reqwest::Error), - #[error("{0}")] - Extension(#[from] axum::extract::rejection::ExtensionRejection), - #[error("{0}")] - Path(#[from] axum::extract::rejection::PathRejection), - #[error("{0}")] - Http(#[from] http::Error), - - // ruma - #[error("{0}")] - IntoHttpError(#[from] ruma::api::error::IntoHttpError), - #[error("{0}")] - RumaError(#[from] ruma::api::client::error::Error), - #[error("uiaa")] - Uiaa(ruma::api::client::uiaa::UiaaInfo), - #[error("{0}")] - Mxid(#[from] ruma::IdParseError), - #[error("{0}: {1}")] - BadRequest(ruma::api::client::error::ErrorKind, &'static str), - #[error("from {0}: {1}")] - Redaction(OwnedServerName, ruma::canonical_json::RedactionError), - #[error("Remote server {0} responded with: {1}")] - Federation(OwnedServerName, ruma::api::client::error::Error), - #[error("{0} in {1}")] - InconsistentRoomState(&'static str, ruma::OwnedRoomId), - - // conduwuit - #[error("Arithmetic operation failed: {0}")] - Arithmetic(&'static str), - #[error("There was a problem with the '{0}' directive in your configuration: {1}")] - Config(&'static str, String), - #[error("{0}")] - BadDatabase(&'static str), - #[error("{0}")] - Database(String), - #[error("{0}")] - BadServerResponse(&'static str), - #[error("{0}")] - Conflict(&'static str), // This is only needed for when a room alias already exists - - // unique / untyped - #[error("{0}")] - Err(String), -} - -impl Error { - pub fn bad_database(message: &'static str) -> Self { - error!("BadDatabase: {}", message); - Self::BadDatabase(message) - } - - #[must_use] - pub fn from_panic(e: Box) -> Self { Self::Panic(panic_str(&e), e) } - - pub fn into_panic(self) -> Box { - match self { - Self::Panic(_, e) | Self::PanicAny(e) => e, - Self::JoinError(e) => e.into_panic(), - _ => std::panic::panic_any(self), - } - } - - /// Sanitizes public-facing errors that can leak sensitive information. - pub fn sanitized_string(&self) -> String { - match self { - Self::Database(..) => String::from("Database error occurred."), - Self::Io(..) => String::from("I/O error occurred."), - _ => self.to_string(), - } - } - - /// Get the panic message string. - pub fn panic_str(self) -> Option<&'static str> { self.is_panic().then_some(panic_str(&self.into_panic())) } - - /// Returns the Matrix error code / error kind - #[inline] - pub fn error_code(&self) -> ruma::api::client::error::ErrorKind { - use ruma::api::client::error::ErrorKind::Unknown; - - match self { - Self::Federation(_, error) => ruma_error_kind(error).clone(), - Self::BadRequest(kind, _) => kind.clone(), - _ => Unknown, - } - } - - /// Check if the Error is trafficking a panic object. - #[inline] - pub fn is_panic(&self) -> bool { - match &self { - Self::Panic(..) | Self::PanicAny(..) => true, - Self::JoinError(e) => e.is_panic(), - _ => false, - } - } -} - -impl UnwindSafe for Error {} -impl RefUnwindSafe for Error {} - -impl From for Error { - #[cold] - #[inline(never)] - fn from(_i: Infallible) -> Self { - panic!("infallible error should never exist"); - } -} - -impl fmt::Debug for Error { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{self}") } -} - -#[inline] -pub fn else_log(error: E) -> Result -where - T: Default, - Error: From, -{ - Ok(default_log(error)) -} - -#[inline] -pub fn else_debug_log(error: E) -> Result -where - T: Default, - Error: From, -{ - Ok(default_debug_log(error)) -} - -#[inline] -pub fn default_log(error: E) -> T -where - T: Default, - Error: From, -{ - let error = Error::from(error); - inspect_log(&error); - T::default() -} - -#[inline] -pub fn default_debug_log(error: E) -> T -where - T: Default, - Error: From, -{ - let error = Error::from(error); - inspect_debug_log(&error); - T::default() -} - -#[inline] -pub fn map_log(error: E) -> Error -where - Error: From, -{ - let error = Error::from(error); - inspect_log(&error); - error -} - -#[inline] -pub fn map_debug_log(error: E) -> Error -where - Error: From, -{ - let error = Error::from(error); - inspect_debug_log(&error); - error -} - -#[inline] -pub fn inspect_log(error: &E) { - error!("{error}"); -} - -#[inline] -pub fn inspect_debug_log(error: &E) { - debug_error!("{error:?}"); -} - -#[cold] -#[inline(never)] -pub fn infallible(_e: &Infallible) { - panic!("infallible error should never exist"); -} - -impl axum::response::IntoResponse for Error { - fn into_response(self) -> axum::response::Response { - let response: UiaaResponse = self.into(); - response - .try_into_http_response::() - .inspect_err(|e| error!("error response error: {e}")) - .map_or_else( - |_| StatusCode::INTERNAL_SERVER_ERROR.into_response(), - |r| r.map(BytesMut::freeze).map(Full::new).into_response(), - ) - } -} - -impl From for UiaaResponse { - fn from(error: Error) -> Self { - if let Error::Uiaa(uiaainfo) = error { - return Self::AuthResponse(uiaainfo); - } - - let kind = match &error { - Error::Federation(_, ref error) | Error::RumaError(ref error) => ruma_error_kind(error), - Error::BadRequest(kind, _) => kind, - _ => &ruma::api::client::error::ErrorKind::Unknown, - }; - - let status_code = match &error { - Error::Federation(_, ref error) | Error::RumaError(ref error) => error.status_code, - Error::BadRequest(ref kind, _) => bad_request_code(kind), - Error::Conflict(_) => StatusCode::CONFLICT, - _ => StatusCode::INTERNAL_SERVER_ERROR, - }; - - let message = match &error { - Error::Federation(ref origin, ref error) => format!("Answer from {origin}: {error}"), - Error::RumaError(ref error) => ruma_error_message(error), - _ => format!("{error}"), - }; - - let body = ruma::api::client::error::ErrorBody::Standard { - kind: kind.clone(), - message, - }; - - Self::MatrixError(ruma::api::client::error::Error { - status_code, - body, - }) - } -} - -fn bad_request_code(kind: &ruma::api::client::error::ErrorKind) -> StatusCode { - use ruma::api::client::error::ErrorKind::*; - - match kind { - GuestAccessForbidden - | ThreepidAuthFailed - | UserDeactivated - | ThreepidDenied - | WrongRoomKeysVersion { - .. - } - | Forbidden { - .. - } => StatusCode::FORBIDDEN, - - UnknownToken { - .. - } - | MissingToken - | Unauthorized => StatusCode::UNAUTHORIZED, - - LimitExceeded { - .. - } => StatusCode::TOO_MANY_REQUESTS, - - TooLarge => StatusCode::PAYLOAD_TOO_LARGE, - - NotFound | Unrecognized => StatusCode::NOT_FOUND, - - _ => StatusCode::BAD_REQUEST, - } -} - -fn ruma_error_message(error: &ruma::api::client::error::Error) -> String { - if let ruma::api::client::error::ErrorBody::Standard { - message, - .. - } = &error.body - { - return message.to_string(); - } - - format!("{error}") -} - -fn ruma_error_kind(e: &ruma::api::client::error::Error) -> &ruma::api::client::error::ErrorKind { - e.error_kind() - .unwrap_or(&ruma::api::client::error::ErrorKind::Unknown) -} diff --git a/src/core/error/err.rs b/src/core/error/err.rs new file mode 100644 index 00000000..12dd4f3b --- /dev/null +++ b/src/core/error/err.rs @@ -0,0 +1,42 @@ +#[macro_export] +macro_rules! Err { + ($($args:tt)*) => { + Err($crate::err!($($args)*)) + }; +} + +#[macro_export] +macro_rules! err { + (error!($($args:tt),+)) => {{ + $crate::error!($($args),+); + $crate::error::Error::Err(std::format!($($args),+)) + }}; + + (debug_error!($($args:tt),+)) => {{ + $crate::debug_error!($($args),+); + $crate::error::Error::Err(std::format!($($args),+)) + }}; + + ($variant:ident(error!($($args:tt),+))) => {{ + $crate::error!($($args),+); + $crate::error::Error::$variant(std::format!($($args),+)) + }}; + + ($variant:ident(debug_error!($($args:tt),+))) => {{ + $crate::debug_error!($($args),+); + $crate::error::Error::$variant(std::format!($($args),+)) + }}; + + (Config($item:literal, $($args:tt),+)) => {{ + $crate::error!(config = %$item, $($args),+); + $crate::error::Error::Config($item, std::format!($($args),+)) + }}; + + ($variant:ident($($args:tt),+)) => { + $crate::error::Error::$variant(std::format!($($args),+)) + }; + + ($string:literal$(,)? $($args:tt),*) => { + $crate::error::Error::Err(std::format!($string, $($args),*)) + }; +} diff --git a/src/core/error/log.rs b/src/core/error/log.rs new file mode 100644 index 00000000..c272bf73 --- /dev/null +++ b/src/core/error/log.rs @@ -0,0 +1,74 @@ +use std::{convert::Infallible, fmt}; + +use super::Error; +use crate::{debug_error, error}; + +#[inline] +pub fn else_log(error: E) -> Result +where + T: Default, + Error: From, +{ + Ok(default_log(error)) +} + +#[inline] +pub fn else_debug_log(error: E) -> Result +where + T: Default, + Error: From, +{ + Ok(default_debug_log(error)) +} + +#[inline] +pub fn default_log(error: E) -> T +where + T: Default, + Error: From, +{ + let error = Error::from(error); + inspect_log(&error); + T::default() +} + +#[inline] +pub fn default_debug_log(error: E) -> T +where + T: Default, + Error: From, +{ + let error = Error::from(error); + inspect_debug_log(&error); + T::default() +} + +#[inline] +pub fn map_log(error: E) -> Error +where + Error: From, +{ + let error = Error::from(error); + inspect_log(&error); + error +} + +#[inline] +pub fn map_debug_log(error: E) -> Error +where + Error: From, +{ + let error = Error::from(error); + inspect_debug_log(&error); + error +} + +#[inline] +pub fn inspect_log(error: &E) { + error!("{error}"); +} + +#[inline] +pub fn inspect_debug_log(error: &E) { + debug_error!("{error:?}"); +} diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs new file mode 100644 index 00000000..cbc22f3e --- /dev/null +++ b/src/core/error/mod.rs @@ -0,0 +1,156 @@ +mod err; +mod log; +mod panic; +mod response; + +use std::{any::Any, convert::Infallible, fmt}; + +pub use log::*; + +use crate::error; + +#[derive(thiserror::Error)] +pub enum Error { + #[error("PANIC!")] + PanicAny(Box), + #[error("PANIC! {0}")] + Panic(&'static str, Box), + + // std + #[error("{0}")] + Fmt(#[from] fmt::Error), + #[error("I/O error: {0}")] + Io(#[from] std::io::Error), + #[error("{0}")] + Utf8Error(#[from] std::str::Utf8Error), + #[error("{0}")] + FromUtf8Error(#[from] std::string::FromUtf8Error), + #[error("{0}")] + TryFromSliceError(#[from] std::array::TryFromSliceError), + #[error("{0}")] + TryFromIntError(#[from] std::num::TryFromIntError), + #[error("{0}")] + ParseIntError(#[from] std::num::ParseIntError), + #[error("{0}")] + ParseFloatError(#[from] std::num::ParseFloatError), + + // third-party + #[error("Join error: {0}")] + JoinError(#[from] tokio::task::JoinError), + #[error("Regex error: {0}")] + Regex(#[from] regex::Error), + #[error("Tracing filter error: {0}")] + TracingFilter(#[from] tracing_subscriber::filter::ParseError), + #[error("Tracing reload error: {0}")] + TracingReload(#[from] tracing_subscriber::reload::Error), + #[error("Image error: {0}")] + Image(#[from] image::error::ImageError), + #[error("Request error: {0}")] + Reqwest(#[from] reqwest::Error), + #[error("{0}")] + Extension(#[from] axum::extract::rejection::ExtensionRejection), + #[error("{0}")] + Path(#[from] axum::extract::rejection::PathRejection), + #[error("{0}")] + Http(#[from] http::Error), + + // ruma + #[error("{0}")] + IntoHttpError(#[from] ruma::api::error::IntoHttpError), + #[error("{0}")] + RumaError(#[from] ruma::api::client::error::Error), + #[error("uiaa")] + Uiaa(ruma::api::client::uiaa::UiaaInfo), + #[error("{0}")] + Mxid(#[from] ruma::IdParseError), + #[error("{0}: {1}")] + BadRequest(ruma::api::client::error::ErrorKind, &'static str), + #[error("from {0}: {1}")] + Redaction(ruma::OwnedServerName, ruma::canonical_json::RedactionError), + #[error("Remote server {0} responded with: {1}")] + Federation(ruma::OwnedServerName, ruma::api::client::error::Error), + #[error("{0} in {1}")] + InconsistentRoomState(&'static str, ruma::OwnedRoomId), + + // conduwuit + #[error("Arithmetic operation failed: {0}")] + Arithmetic(&'static str), + #[error("There was a problem with the '{0}' directive in your configuration: {1}")] + Config(&'static str, String), + #[error("{0}")] + BadDatabase(&'static str), + #[error("{0}")] + Database(String), + #[error("{0}")] + BadServerResponse(&'static str), + #[error("{0}")] + Conflict(&'static str), // This is only needed for when a room alias already exists + + // unique / untyped + #[error("{0}")] + Err(String), +} + +impl Error { + pub fn bad_database(message: &'static str) -> Self { + error!("BadDatabase: {}", message); + Self::BadDatabase(message) + } + + /// Sanitizes public-facing errors that can leak sensitive information. + pub fn sanitized_string(&self) -> String { + match self { + Self::Database(..) => String::from("Database error occurred."), + Self::Io(..) => String::from("I/O error occurred."), + _ => self.to_string(), + } + } + + pub fn message(&self) -> String { + match self { + Self::Federation(ref origin, ref error) => format!("Answer from {origin}: {error}"), + Self::RumaError(ref error) => response::ruma_error_message(error), + _ => format!("{self}"), + } + } + + /// Returns the Matrix error code / error kind + #[inline] + pub fn kind(&self) -> ruma::api::client::error::ErrorKind { + use ruma::api::client::error::ErrorKind::Unknown; + + match self { + Self::Federation(_, error) => response::ruma_error_kind(error).clone(), + Self::BadRequest(kind, _) => kind.clone(), + _ => Unknown, + } + } + + pub fn status_code(&self) -> http::StatusCode { + match self { + Self::Federation(_, ref error) | Self::RumaError(ref error) => error.status_code, + Self::BadRequest(ref kind, _) => response::bad_request_code(kind), + Self::Conflict(_) => http::StatusCode::CONFLICT, + _ => http::StatusCode::INTERNAL_SERVER_ERROR, + } + } +} + +impl fmt::Debug for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{self}") } +} + +#[allow(clippy::fallible_impl_from)] +impl From for Error { + #[cold] + #[inline(never)] + fn from(_e: Infallible) -> Self { + panic!("infallible error should never exist"); + } +} + +#[cold] +#[inline(never)] +pub fn infallible(_e: &Infallible) { + panic!("infallible error should never exist"); +} diff --git a/src/core/error/panic.rs b/src/core/error/panic.rs new file mode 100644 index 00000000..c070f786 --- /dev/null +++ b/src/core/error/panic.rs @@ -0,0 +1,41 @@ +use std::{ + any::Any, + panic::{panic_any, RefUnwindSafe, UnwindSafe}, +}; + +use super::Error; +use crate::debug; + +impl UnwindSafe for Error {} +impl RefUnwindSafe for Error {} + +impl Error { + pub fn panic(self) -> ! { panic_any(self.into_panic()) } + + #[must_use] + pub fn from_panic(e: Box) -> Self { Self::Panic(debug::panic_str(&e), e) } + + pub fn into_panic(self) -> Box { + match self { + Self::Panic(_, e) | Self::PanicAny(e) => e, + Self::JoinError(e) => e.into_panic(), + _ => Box::new(self), + } + } + + /// Get the panic message string. + pub fn panic_str(self) -> Option<&'static str> { + self.is_panic() + .then_some(debug::panic_str(&self.into_panic())) + } + + /// Check if the Error is trafficking a panic object. + #[inline] + pub fn is_panic(&self) -> bool { + match &self { + Self::Panic(..) | Self::PanicAny(..) => true, + Self::JoinError(e) => e.is_panic(), + _ => false, + } + } +} diff --git a/src/core/error/response.rs b/src/core/error/response.rs new file mode 100644 index 00000000..4ea76e26 --- /dev/null +++ b/src/core/error/response.rs @@ -0,0 +1,88 @@ +use bytes::BytesMut; +use http::StatusCode; +use http_body_util::Full; +use ruma::api::{client::uiaa::UiaaResponse, OutgoingResponse}; + +use super::Error; +use crate::error; + +impl axum::response::IntoResponse for Error { + fn into_response(self) -> axum::response::Response { + let response: UiaaResponse = self.into(); + response + .try_into_http_response::() + .inspect_err(|e| error!("error response error: {e}")) + .map_or_else( + |_| StatusCode::INTERNAL_SERVER_ERROR.into_response(), + |r| r.map(BytesMut::freeze).map(Full::new).into_response(), + ) + } +} + +impl From for UiaaResponse { + fn from(error: Error) -> Self { + if let Error::Uiaa(uiaainfo) = error { + return Self::AuthResponse(uiaainfo); + } + + let body = ruma::api::client::error::ErrorBody::Standard { + kind: error.kind(), + message: error.message(), + }; + + Self::MatrixError(ruma::api::client::error::Error { + status_code: error.status_code(), + body, + }) + } +} + +pub(super) fn bad_request_code(kind: &ruma::api::client::error::ErrorKind) -> StatusCode { + use ruma::api::client::error::ErrorKind::*; + + match kind { + GuestAccessForbidden + | ThreepidAuthFailed + | UserDeactivated + | ThreepidDenied + | WrongRoomKeysVersion { + .. + } + | Forbidden { + .. + } => StatusCode::FORBIDDEN, + + UnknownToken { + .. + } + | MissingToken + | Unauthorized => StatusCode::UNAUTHORIZED, + + LimitExceeded { + .. + } => StatusCode::TOO_MANY_REQUESTS, + + TooLarge => StatusCode::PAYLOAD_TOO_LARGE, + + NotFound | Unrecognized => StatusCode::NOT_FOUND, + + _ => StatusCode::BAD_REQUEST, + } +} + +pub(super) fn ruma_error_message(error: &ruma::api::client::error::Error) -> String { + if let ruma::api::client::error::ErrorBody::Standard { + message, + .. + } = &error.body + { + return message.to_string(); + } + + format!("{error}") +} + +pub(super) fn ruma_error_kind(e: &ruma::api::client::error::Error) -> &ruma::api::client::error::ErrorKind { + e.error_kind() + .unwrap_or(&ruma::api::client::error::ErrorKind::Unknown) +} From b8baa1223d8083b48e1127bb29378eed32033c7e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 12 Jul 2024 01:08:53 +0000 Subject: [PATCH 113/158] reduce RoomVersionId patterns Signed-off-by: Jason Volk --- src/api/client/membership.rs | 11 +--- src/api/client/room.rs | 85 ++++++++++---------------- src/api/server/make_join.rs | 57 ++++++++--------- src/api/server/make_leave.rs | 26 ++------ src/service/admin/create.rs | 33 +++++----- src/service/rooms/event_handler/mod.rs | 79 +++++++++++------------- src/service/rooms/timeline/mod.rs | 27 ++------ 7 files changed, 121 insertions(+), 197 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index c25c08d1..3adee631 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -780,14 +780,9 @@ async fn join_room_by_id_helper_remote( info!("send_join finished"); if join_authorized_via_users_server.is_some() { + use RoomVersionId::*; match &room_version_id { - RoomVersionId::V1 - | RoomVersionId::V2 - | RoomVersionId::V3 - | RoomVersionId::V4 - | RoomVersionId::V5 - | RoomVersionId::V6 - | RoomVersionId::V7 => { + V1 | V2 | V3 | V4 | V5 | V6 | V7 => { warn!( "Found `join_authorised_via_users_server` but room {} is version {}. Ignoring.", room_id, &room_version_id @@ -795,7 +790,7 @@ async fn join_room_by_id_helper_remote( }, // only room versions 8 and above using `join_authorized_via_users_server` (restricted joins) need to // validate and send signatures - RoomVersionId::V8 | RoomVersionId::V9 | RoomVersionId::V10 | RoomVersionId::V11 => { + V8 | V9 | V10 | V11 => { if let Some(signed_raw) = &send_join_response.room_state.event { info!( "There is a signed event. This room is probably using restricted joins. Adding signature to \ diff --git a/src/api/client/room.rs b/src/api/client/room.rs index ccdf4dc0..adf58b04 100644 --- a/src/api/client/room.rs +++ b/src/api/client/room.rs @@ -118,6 +118,8 @@ pub(crate) async fn create_room_route(body: Ruma) -> R let content = match &body.creation_content { Some(content) => { + use RoomVersionId::*; + let mut content = content .deserialize_as::() .map_err(|e| { @@ -125,16 +127,7 @@ pub(crate) async fn create_room_route(body: Ruma) -> R Error::bad_database("Failed to deserialise content as canonical JSON.") })?; match room_version { - RoomVersionId::V1 - | RoomVersionId::V2 - | RoomVersionId::V3 - | RoomVersionId::V4 - | RoomVersionId::V5 - | RoomVersionId::V6 - | RoomVersionId::V7 - | RoomVersionId::V8 - | RoomVersionId::V9 - | RoomVersionId::V10 => { + V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { content.insert( "creator".into(), json!(&sender_user).try_into().map_err(|e| { @@ -143,7 +136,7 @@ pub(crate) async fn create_room_route(body: Ruma) -> R })?, ); }, - RoomVersionId::V11 => {}, // V11 removed the "creator" key + V11 => {}, // V11 removed the "creator" key _ => { warn!("Unexpected or unsupported room version {room_version}"); return Err(Error::BadRequest( @@ -152,7 +145,6 @@ pub(crate) async fn create_room_route(body: Ruma) -> R )); }, } - content.insert( "room_version".into(), json!(room_version.as_str()) @@ -162,18 +154,11 @@ pub(crate) async fn create_room_route(body: Ruma) -> R content }, None => { + use RoomVersionId::*; + let content = match room_version { - RoomVersionId::V1 - | RoomVersionId::V2 - | RoomVersionId::V3 - | RoomVersionId::V4 - | RoomVersionId::V5 - | RoomVersionId::V6 - | RoomVersionId::V7 - | RoomVersionId::V8 - | RoomVersionId::V9 - | RoomVersionId::V10 => RoomCreateEventContent::new_v1(sender_user.clone()), - RoomVersionId::V11 => RoomCreateEventContent::new_v11(), + V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => RoomCreateEventContent::new_v1(sender_user.clone()), + V11 => RoomCreateEventContent::new_v11(), _ => { warn!("Unexpected or unsupported room version {room_version}"); return Err(Error::BadRequest( @@ -623,36 +608,30 @@ pub(crate) async fn upgrade_room_route(body: Ruma) -> // Send a m.room.create event containing a predecessor field and the applicable // room_version - match body.new_version { - RoomVersionId::V1 - | RoomVersionId::V2 - | RoomVersionId::V3 - | RoomVersionId::V4 - | RoomVersionId::V5 - | RoomVersionId::V6 - | RoomVersionId::V7 - | RoomVersionId::V8 - | RoomVersionId::V9 - | RoomVersionId::V10 => { - create_event_content.insert( - "creator".into(), - json!(&sender_user).try_into().map_err(|e| { - info!("Error forming creation event: {e}"); - Error::BadRequest(ErrorKind::BadJson, "Error forming creation event") - })?, - ); - }, - RoomVersionId::V11 => { - // "creator" key no longer exists in V11 rooms - create_event_content.remove("creator"); - }, - _ => { - warn!("Unexpected or unsupported room version {}", body.new_version); - return Err(Error::BadRequest( - ErrorKind::BadJson, - "Unexpected or unsupported room version found", - )); - }, + { + use RoomVersionId::*; + match body.new_version { + V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { + create_event_content.insert( + "creator".into(), + json!(&sender_user).try_into().map_err(|e| { + info!("Error forming creation event: {e}"); + Error::BadRequest(ErrorKind::BadJson, "Error forming creation event") + })?, + ); + }, + V11 => { + // "creator" key no longer exists in V11 rooms + create_event_content.remove("creator"); + }, + _ => { + warn!("Unexpected or unsupported room version {}", body.new_version); + return Err(Error::BadRequest( + ErrorKind::BadJson, + "Unexpected or unsupported room version found", + )); + }, + } } create_event_content.insert( diff --git a/src/api/server/make_join.rs b/src/api/server/make_join.rs index c909ea33..b5dadf7f 100644 --- a/src/api/server/make_join.rs +++ b/src/api/server/make_join.rs @@ -7,7 +7,7 @@ use ruma::{ }, StateEventType, TimelineEventType, }, - RoomId, RoomVersionId, UserId, + CanonicalJsonObject, RoomId, RoomVersionId, UserId, }; use serde_json::value::to_raw_value; use tracing::warn; @@ -144,27 +144,7 @@ pub(crate) async fn create_join_event_template_route( drop(state_lock); // room v3 and above removed the "event_id" field from remote PDU format - match room_version_id { - RoomVersionId::V1 | RoomVersionId::V2 => {}, - RoomVersionId::V3 - | RoomVersionId::V4 - | RoomVersionId::V5 - | RoomVersionId::V6 - | RoomVersionId::V7 - | RoomVersionId::V8 - | RoomVersionId::V9 - | RoomVersionId::V10 - | RoomVersionId::V11 => { - pdu_json.remove("event_id"); - }, - _ => { - warn!("Unexpected or unsupported room version {room_version_id}"); - return Err(Error::BadRequest( - ErrorKind::BadJson, - "Unexpected or unsupported room version found", - )); - }, - }; + maybe_strip_event_id(&mut pdu_json, &room_version_id)?; Ok(prepare_join_event::v1::Response { room_version: Some(room_version_id), @@ -179,6 +159,8 @@ pub(crate) async fn create_join_event_template_route( pub(crate) fn user_can_perform_restricted_join( user_id: &UserId, room_id: &RoomId, room_version_id: &RoomVersionId, ) -> Result { + use RoomVersionId::*; + let join_rules_event = services() .rooms @@ -198,16 +180,7 @@ pub(crate) fn user_can_perform_restricted_join( return Ok(false); }; - if matches!( - room_version_id, - RoomVersionId::V1 - | RoomVersionId::V2 - | RoomVersionId::V3 - | RoomVersionId::V4 - | RoomVersionId::V5 - | RoomVersionId::V6 - | RoomVersionId::V7 - ) { + if matches!(room_version_id, V1 | V2 | V3 | V4 | V5 | V6 | V7) { return Ok(false); } @@ -239,3 +212,23 @@ pub(crate) fn user_can_perform_restricted_join( )) } } + +pub(crate) fn maybe_strip_event_id(pdu_json: &mut CanonicalJsonObject, room_version_id: &RoomVersionId) -> Result<()> { + use RoomVersionId::*; + + match room_version_id { + V1 | V2 => {}, + V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 | V11 => { + pdu_json.remove("event_id"); + }, + _ => { + warn!("Unexpected or unsupported room version {room_version_id}"); + return Err(Error::BadRequest( + ErrorKind::BadJson, + "Unexpected or unsupported room version found", + )); + }, + }; + + Ok(()) +} diff --git a/src/api/server/make_leave.rs b/src/api/server/make_leave.rs index eea3e4f8..63fc2b2e 100644 --- a/src/api/server/make_leave.rs +++ b/src/api/server/make_leave.rs @@ -1,14 +1,15 @@ +use conduit::{Error, Result}; use ruma::{ api::{client::error::ErrorKind, federation::membership::prepare_leave_event}, events::{ room::member::{MembershipState, RoomMemberEventContent}, TimelineEventType, }, - RoomVersionId, }; use serde_json::value::to_raw_value; -use crate::{service::pdu::PduBuilder, services, Error, Result, Ruma}; +use super::make_join::maybe_strip_event_id; +use crate::{service::pdu::PduBuilder, services, Ruma}; /// # `PUT /_matrix/federation/v1/make_leave/{roomId}/{eventId}` /// @@ -64,26 +65,7 @@ pub(crate) async fn create_leave_event_template_route( drop(state_lock); // room v3 and above removed the "event_id" field from remote PDU format - match room_version_id { - RoomVersionId::V1 | RoomVersionId::V2 => {}, - RoomVersionId::V3 - | RoomVersionId::V4 - | RoomVersionId::V5 - | RoomVersionId::V6 - | RoomVersionId::V7 - | RoomVersionId::V8 - | RoomVersionId::V9 - | RoomVersionId::V10 - | RoomVersionId::V11 => { - pdu_json.remove("event_id"); - }, - _ => { - return Err(Error::BadRequest( - ErrorKind::BadJson, - "Unexpected or unsupported room version found", - )); - }, - }; + maybe_strip_event_id(&mut pdu_json, &room_version_id)?; Ok(prepare_leave_event::v1::Response { room_version: Some(room_version_id), diff --git a/src/service/admin/create.rs b/src/service/admin/create.rs index 57df344a..fbb6a078 100644 --- a/src/service/admin/create.rs +++ b/src/service/admin/create.rs @@ -41,25 +41,20 @@ pub async fn create_admin_room() -> Result<()> { services().users.create(server_user, None)?; let room_version = services().globals.default_room_version(); - let mut content = match room_version { - RoomVersionId::V1 - | RoomVersionId::V2 - | RoomVersionId::V3 - | RoomVersionId::V4 - | RoomVersionId::V5 - | RoomVersionId::V6 - | RoomVersionId::V7 - | RoomVersionId::V8 - | RoomVersionId::V9 - | RoomVersionId::V10 => RoomCreateEventContent::new_v1(server_user.clone()), - RoomVersionId::V11 => RoomCreateEventContent::new_v11(), - _ => { - warn!("Unexpected or unsupported room version {}", room_version); - return Err(Error::BadRequest( - ErrorKind::BadJson, - "Unexpected or unsupported room version found", - )); - }, + + let mut content = { + use RoomVersionId::*; + match room_version { + V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => RoomCreateEventContent::new_v1(server_user.clone()), + V11 => RoomCreateEventContent::new_v11(), + _ => { + warn!("Unexpected or unsupported room version {}", room_version); + return Err(Error::BadRequest( + ErrorKind::BadJson, + "Unexpected or unsupported room version found", + )); + }, + } }; content.federate = true; diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 9490a404..33c00643 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -551,49 +551,44 @@ impl Service { // Soft fail check before doing state res debug!("Performing soft-fail check"); - let soft_fail = !state_res::event_auth::auth_check(&room_version, &incoming_pdu, None::, |k, s| { - auth_events.get(&(k.clone(), s.to_owned())) - }) - .map_err(|_e| Error::BadRequest(ErrorKind::forbidden(), "Auth check failed."))? - || incoming_pdu.kind == TimelineEventType::RoomRedaction - && match room_version_id { - RoomVersionId::V1 - | RoomVersionId::V2 - | RoomVersionId::V3 - | RoomVersionId::V4 - | RoomVersionId::V5 - | RoomVersionId::V6 - | RoomVersionId::V7 - | RoomVersionId::V8 - | RoomVersionId::V9 - | RoomVersionId::V10 => { - if let Some(redact_id) = &incoming_pdu.redacts { - !services().rooms.state_accessor.user_can_redact( - redact_id, - &incoming_pdu.sender, - &incoming_pdu.room_id, - true, - )? - } else { - false - } - }, - _ => { - let content = serde_json::from_str::(incoming_pdu.content.get()) - .map_err(|_| Error::bad_database("Invalid content in redaction pdu."))?; + let soft_fail = { + use RoomVersionId::*; - if let Some(redact_id) = &content.redacts { - !services().rooms.state_accessor.user_can_redact( - redact_id, - &incoming_pdu.sender, - &incoming_pdu.room_id, - true, - )? - } else { - false - } - }, - }; + !state_res::event_auth::auth_check(&room_version, &incoming_pdu, None::, |k, s| { + auth_events.get(&(k.clone(), s.to_owned())) + }) + .map_err(|_e| Error::BadRequest(ErrorKind::forbidden(), "Auth check failed."))? + || incoming_pdu.kind == TimelineEventType::RoomRedaction + && match room_version_id { + V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { + if let Some(redact_id) = &incoming_pdu.redacts { + !services().rooms.state_accessor.user_can_redact( + redact_id, + &incoming_pdu.sender, + &incoming_pdu.room_id, + true, + )? + } else { + false + } + }, + _ => { + let content = serde_json::from_str::(incoming_pdu.content.get()) + .map_err(|_| Error::bad_database("Invalid content in redaction pdu."))?; + + if let Some(redact_id) = &content.redacts { + !services().rooms.state_accessor.user_can_redact( + redact_id, + &incoming_pdu.sender, + &incoming_pdu.room_id, + true, + )? + } else { + false + } + }, + } + }; // 13. Use state resolution to find new room state diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index e255e978..0bc5ade1 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -389,18 +389,11 @@ impl Service { match pdu.kind { TimelineEventType::RoomRedaction => { + use RoomVersionId::*; + let room_version_id = services().rooms.state.get_room_version(&pdu.room_id)?; match room_version_id { - RoomVersionId::V1 - | RoomVersionId::V2 - | RoomVersionId::V3 - | RoomVersionId::V4 - | RoomVersionId::V5 - | RoomVersionId::V6 - | RoomVersionId::V7 - | RoomVersionId::V8 - | RoomVersionId::V9 - | RoomVersionId::V10 => { + V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { if let Some(redact_id) = &pdu.redacts { if services().rooms.state_accessor.user_can_redact( redact_id, @@ -412,7 +405,7 @@ impl Service { } } }, - RoomVersionId::V11 => { + V11 => { let content = serde_json::from_str::(pdu.content.get()).map_err(|e| { warn!("Invalid content in redaction pdu: {e}"); @@ -868,17 +861,9 @@ impl Service { // If redaction event is not authorized, do not append it to the timeline if pdu.kind == TimelineEventType::RoomRedaction { + use RoomVersionId::*; match services().rooms.state.get_room_version(&pdu.room_id)? { - RoomVersionId::V1 - | RoomVersionId::V2 - | RoomVersionId::V3 - | RoomVersionId::V4 - | RoomVersionId::V5 - | RoomVersionId::V6 - | RoomVersionId::V7 - | RoomVersionId::V8 - | RoomVersionId::V9 - | RoomVersionId::V10 => { + V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { if let Some(redact_id) = &pdu.redacts { if !services().rooms.state_accessor.user_can_redact( redact_id, From 5be679e17b67e22f8a91fb2906be091198f4c713 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 13 Jul 2024 06:07:09 +0000 Subject: [PATCH 114/158] refactor main task stack through service mgr Signed-off-by: Jason Volk --- src/router/run.rs | 54 ++++++++++++++++++++++++++++--------- src/router/serve/mod.rs | 16 +++++------ src/service/mod.rs | 4 +-- src/service/services.rs | 60 ++++++++++++++++++++++++++++------------- 4 files changed, 93 insertions(+), 41 deletions(-) diff --git a/src/router/run.rs b/src/router/run.rs index 02cec781..b3ec2285 100644 --- a/src/router/run.rs +++ b/src/router/run.rs @@ -1,8 +1,10 @@ use std::{sync::Arc, time::Duration}; use axum_server::Handle as ServerHandle; -use tokio::sync::broadcast::{self, Sender}; -use tracing::{debug, error, info}; +use tokio::{ + sync::broadcast::{self, Sender}, + task::JoinHandle, +}; extern crate conduit_admin as admin; extern crate conduit_core as conduit; @@ -10,14 +12,14 @@ extern crate conduit_service as service; use std::sync::atomic::Ordering; -use conduit::{debug_info, trace, Error, Result, Server}; +use conduit::{debug, debug_info, error, info, trace, Error, Result, Server}; -use crate::{layers, serve}; +use crate::serve; /// Main loop base #[tracing::instrument(skip_all)] -pub(crate) async fn run(server: Arc) -> Result<(), Error> { - let app = layers::build(&server)?; +pub(crate) async fn run(server: Arc) -> Result<()> { + debug!("Start"); // Install the admin room callback here for now admin::init().await; @@ -29,8 +31,16 @@ pub(crate) async fn run(server: Arc) -> Result<(), Error> { .runtime() .spawn(signal(server.clone(), tx.clone(), handle.clone())); - // Serve clients - let res = serve::serve(&server, app, handle, tx.subscribe()).await; + let mut listener = server + .runtime() + .spawn(serve::serve(server.clone(), handle.clone(), tx.subscribe())); + + // Focal point + debug!("Running"); + let res = tokio::select! { + res = &mut listener => res.map_err(Error::from).unwrap_or_else(Err), + res = service::services().poll() => handle_services_poll(&server, res, listener).await, + }; // Join the signal handler before we leave. sigs.abort(); @@ -39,16 +49,16 @@ pub(crate) async fn run(server: Arc) -> Result<(), Error> { // Remove the admin room callback admin::fini().await; - debug_info!("Finished"); + debug_info!("Finish"); res } /// Async initializations #[tracing::instrument(skip_all)] -pub(crate) async fn start(server: Arc) -> Result<(), Error> { +pub(crate) async fn start(server: Arc) -> Result<()> { debug!("Starting..."); - service::init(&server).await?; + service::start(&server).await?; #[cfg(feature = "systemd")] sd_notify::notify(true, &[sd_notify::NotifyState::Ready]).expect("failed to notify systemd of ready state"); @@ -59,12 +69,12 @@ pub(crate) async fn start(server: Arc) -> Result<(), Error> { /// Async destructions #[tracing::instrument(skip_all)] -pub(crate) async fn stop(_server: Arc) -> Result<(), Error> { +pub(crate) async fn stop(_server: Arc) -> Result<()> { debug!("Shutting down..."); // Wait for all completions before dropping or we'll lose them to the module // unload and explode. - service::fini().await; + service::stop().await; debug!("Cleaning up..."); @@ -108,3 +118,21 @@ async fn handle_shutdown(server: &Arc, tx: &Sender<()>, handle: &axum_se handle.shutdown(); } } + +async fn handle_services_poll( + server: &Arc, result: Result<()>, listener: JoinHandle>, +) -> Result<()> { + debug!("Service manager finished: {result:?}"); + + if server.running() { + if let Err(e) = server.shutdown() { + error!("Failed to send shutdown signal: {e}"); + } + } + + if let Err(e) = listener.await { + error!("Client listener task finished with error: {e}"); + } + + result +} diff --git a/src/router/serve/mod.rs b/src/router/serve/mod.rs index 47f2fd43..4e923444 100644 --- a/src/router/serve/mod.rs +++ b/src/router/serve/mod.rs @@ -4,23 +4,23 @@ mod unix; use std::sync::Arc; -use axum::Router; use axum_server::Handle as ServerHandle; -use conduit::{Error, Result, Server}; +use conduit::{Result, Server}; use tokio::sync::broadcast; +use crate::layers; + /// Serve clients -pub(super) async fn serve( - server: &Arc, app: Router, handle: ServerHandle, shutdown: broadcast::Receiver<()>, -) -> Result<(), Error> { +pub(super) async fn serve(server: Arc, handle: ServerHandle, shutdown: broadcast::Receiver<()>) -> Result<()> { let config = &server.config; let addrs = config.get_bind_addrs(); + let app = layers::build(&server)?; if cfg!(unix) && config.unix_socket_path.is_some() { - unix::serve(server, app, shutdown).await + unix::serve(&server, app, shutdown).await } else if config.tls.is_some() { - tls::serve(server, app, handle, addrs).await + tls::serve(&server, app, handle, addrs).await } else { - plain::serve(server, app, handle, addrs).await + plain::serve(&server, app, handle, addrs).await } } diff --git a/src/service/mod.rs b/src/service/mod.rs index 15c4cc35..3a1eaf57 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -37,7 +37,7 @@ conduit::mod_dtor! {} static SERVICES: RwLock> = RwLock::new(None); -pub async fn init(server: &Arc) -> Result<()> { +pub async fn start(server: &Arc) -> Result<()> { let d = Arc::new(Database::open(server).await?); let s = Box::new(Services::build(server.clone(), d)?); _ = SERVICES.write().expect("write locked").insert(Box::leak(s)); @@ -45,7 +45,7 @@ pub async fn init(server: &Arc) -> Result<()> { services().start().await } -pub async fn fini() { +pub async fn stop() { services().stop().await; // Deactivate services(). Any further use will panic the caller. diff --git a/src/service/services.rs b/src/service/services.rs index 13689008..e9629ef7 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -1,6 +1,6 @@ -use std::{collections::BTreeMap, fmt::Write, panic::AssertUnwindSafe, sync::Arc, time::Duration}; +use std::{collections::BTreeMap, fmt::Write, ops::DerefMut, panic::AssertUnwindSafe, sync::Arc, time::Duration}; -use conduit::{debug, debug_info, error, info, trace, utils::time, warn, Error, Result, Server}; +use conduit::{debug, debug_info, debug_warn, error, info, trace, utils::time, warn, Error, Result, Server}; use database::Database; use futures_util::FutureExt; use tokio::{ @@ -30,8 +30,8 @@ pub struct Services { pub media: Arc, pub sending: Arc, - workers: Mutex, manager: Mutex>>>, + workers: Mutex, pub(crate) service: Map, pub server: Arc, pub db: Arc, @@ -93,15 +93,15 @@ impl Services { media: build!(media::Service), sending: build!(sending::Service), globals: build!(globals::Service), - workers: Mutex::new(JoinSet::new()), manager: Mutex::new(None), + workers: Mutex::new(JoinSet::new()), service, server, db, }) } - pub async fn start(&self) -> Result<()> { + pub(super) async fn start(&self) -> Result<()> { debug_info!("Starting services..."); self.media.create_media_dir().await?; @@ -114,9 +114,7 @@ impl Services { } debug!("Starting service manager..."); - let manager = async move { crate::services().manager().await }; - let manager = self.server.runtime().spawn(manager); - _ = self.manager.lock().await.insert(manager); + self.manager_start().await?; if self.globals.allow_check_for_updates() { let handle = globals::updates::start_check_for_updates_task(); @@ -127,7 +125,7 @@ impl Services { Ok(()) } - pub async fn stop(&self) { + pub(super) async fn stop(&self) { info!("Shutting down services..."); self.interrupt(); @@ -138,15 +136,20 @@ impl Services { } debug!("Stopping service manager..."); - if let Some(manager) = self.manager.lock().await.take() { - if let Err(e) = manager.await { - error!("Manager shutdown error: {e:?}"); - } - } + self.manager_stop().await; debug_info!("Services shutdown complete."); } + pub async fn poll(&self) -> Result<()> { + if let Some(manager) = self.manager.lock().await.deref_mut() { + trace!("Polling service manager..."); + return manager.await?; + } + + Ok(()) + } + pub async fn clear_cache(&self) { for service in self.service.values() { service.clear_cache(); @@ -188,6 +191,26 @@ impl Services { } } + async fn manager_start(&self) -> Result<()> { + debug!("Starting service manager..."); + self.manager.lock().await.get_or_insert_with(|| { + self.server + .runtime() + .spawn(async move { crate::services().manager().await }) + }); + + Ok(()) + } + + async fn manager_stop(&self) { + if let Some(manager) = self.manager.lock().await.take() { + debug!("Waiting for service manager..."); + if let Err(e) = manager.await { + error!("Manager shutdown error: {e:?}"); + } + } + } + async fn manager(&self) -> Result<()> { loop { let mut workers = self.workers.lock().await; @@ -226,14 +249,15 @@ impl Services { &self, workers: &mut WorkersLocked<'_>, service: &Arc, error: Error, ) -> Result<()> { let name = service.name(); - error!("service {name:?} worker error: {error}"); + error!("service {name:?} aborted: {error}"); - if !error.is_panic() { + if !self.server.running() { + debug_warn!("service {name:?} error ignored on shutdown."); return Ok(()); } - if !self.server.running() { - return Ok(()); + if !error.is_panic() { + return Err(error); } let delay = Duration::from_millis(RESTART_DELAY_MS); From 38c989a07efa5670c9690dc064e3f4955758c43d Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 13 Jul 2024 07:01:45 +0000 Subject: [PATCH 115/158] split service manager into unit Signed-off-by: Jason Volk --- src/service/manager.rs | 159 ++++++++++++++++++++++++++++++++++++++++ src/service/mod.rs | 1 + src/service/services.rs | 156 +++++---------------------------------- 3 files changed, 179 insertions(+), 137 deletions(-) create mode 100644 src/service/manager.rs diff --git a/src/service/manager.rs b/src/service/manager.rs new file mode 100644 index 00000000..84a28a76 --- /dev/null +++ b/src/service/manager.rs @@ -0,0 +1,159 @@ +use std::{panic::AssertUnwindSafe, sync::Arc, time::Duration}; + +use conduit::{debug, debug_warn, error, trace, utils::time, warn, Error, Result, Server}; +use futures_util::FutureExt; +use tokio::{ + sync::{Mutex, MutexGuard}, + task::{JoinHandle, JoinSet}, + time::sleep, +}; + +use crate::{service::Service, Services}; + +pub(crate) struct Manager { + manager: Mutex>>>, + workers: Mutex, + server: Arc, + services: &'static Services, +} + +type Workers = JoinSet; +type WorkerResult = (Arc, Result<()>); +type WorkersLocked<'a> = MutexGuard<'a, Workers>; + +const RESTART_DELAY_MS: u64 = 2500; + +impl Manager { + pub(super) fn new(services: &Services) -> Arc { + Arc::new(Self { + manager: Mutex::new(None), + workers: Mutex::new(JoinSet::new()), + server: services.server.clone(), + services: crate::services(), + }) + } + + pub(super) async fn poll(&self) -> Result<()> { + if let Some(manager) = &mut *self.manager.lock().await { + trace!("Polling service manager..."); + return manager.await?; + } + + Ok(()) + } + + pub(super) async fn start(self: Arc) -> Result<()> { + let mut workers = self.workers.lock().await; + + debug!("Starting service manager..."); + let self_ = self.clone(); + _ = self.manager.lock().await.insert( + self.server + .runtime() + .spawn(async move { self_.worker().await }), + ); + + debug!("Starting service workers..."); + for service in self.services.service.values() { + self.start_worker(&mut workers, service).await?; + } + + Ok(()) + } + + pub(super) async fn stop(&self) { + if let Some(manager) = self.manager.lock().await.take() { + debug!("Waiting for service manager..."); + if let Err(e) = manager.await { + error!("Manager shutdown error: {e:?}"); + } + } + } + + async fn worker(&self) -> Result<()> { + loop { + let mut workers = self.workers.lock().await; + tokio::select! { + result = workers.join_next() => match result { + Some(Ok(result)) => self.handle_result(&mut workers, result).await?, + Some(Err(error)) => self.handle_abort(&mut workers, Error::from(error)).await?, + None => break, + } + } + } + + debug!("Worker manager finished"); + Ok(()) + } + + async fn handle_abort(&self, _workers: &mut WorkersLocked<'_>, error: Error) -> Result<()> { + // not supported until service can be associated with abort + unimplemented!("unexpected worker task abort {error:?}"); + } + + async fn handle_result(&self, workers: &mut WorkersLocked<'_>, result: WorkerResult) -> Result<()> { + let (service, result) = result; + match result { + Ok(()) => self.handle_finished(workers, &service).await, + Err(error) => self.handle_error(workers, &service, error).await, + } + } + + async fn handle_finished(&self, _workers: &mut WorkersLocked<'_>, service: &Arc) -> Result<()> { + debug!("service {:?} worker finished", service.name()); + Ok(()) + } + + async fn handle_error( + &self, workers: &mut WorkersLocked<'_>, service: &Arc, error: Error, + ) -> Result<()> { + let name = service.name(); + error!("service {name:?} aborted: {error}"); + + if !self.server.running() { + debug_warn!("service {name:?} error ignored on shutdown."); + return Ok(()); + } + + if !error.is_panic() { + return Err(error); + } + + let delay = Duration::from_millis(RESTART_DELAY_MS); + warn!("service {name:?} worker restarting after {} delay", time::pretty(delay)); + sleep(delay).await; + + self.start_worker(workers, service).await + } + + /// Start the worker in a task for the service. + async fn start_worker(&self, workers: &mut WorkersLocked<'_>, service: &Arc) -> Result<()> { + if !self.server.running() { + return Err(Error::Err(format!( + "Service {:?} worker not starting during server shutdown.", + service.name() + ))); + } + + debug!("Service {:?} worker starting...", service.name()); + workers.spawn_on(worker(service.clone()), self.server.runtime()); + + Ok(()) + } +} + +/// Base frame for service worker. This runs in a tokio::task. All errors and +/// panics from the worker are caught and returned cleanly. The JoinHandle +/// should never error with a panic, and if so it should propagate, but it may +/// error with an Abort which the manager should handle along with results to +/// determine if the worker should be restarted. +async fn worker(service: Arc) -> WorkerResult { + let service_ = Arc::clone(&service); + let result = AssertUnwindSafe(service_.worker()) + .catch_unwind() + .await + .map_err(Error::from_panic); + + // flattens JoinError for panic into worker's Error + (service, result.unwrap_or_else(Err)) +} diff --git a/src/service/mod.rs b/src/service/mod.rs index 3a1eaf57..ba68fae2 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -1,5 +1,6 @@ #![allow(refining_impl_trait)] +mod manager; mod service; pub mod services; diff --git a/src/service/services.rs b/src/service/services.rs index e9629ef7..62e73fd1 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -1,16 +1,13 @@ -use std::{collections::BTreeMap, fmt::Write, ops::DerefMut, panic::AssertUnwindSafe, sync::Arc, time::Duration}; +use std::{collections::BTreeMap, fmt::Write, sync::Arc}; -use conduit::{debug, debug_info, debug_warn, error, info, trace, utils::time, warn, Error, Result, Server}; +use conduit::{debug, debug_info, info, trace, Result, Server}; use database::Database; -use futures_util::FutureExt; -use tokio::{ - sync::{Mutex, MutexGuard}, - task::{JoinHandle, JoinSet}, - time::sleep, -}; +use tokio::sync::Mutex; use crate::{ - account_data, admin, appservice, globals, key_backups, media, presence, pusher, rooms, sending, + account_data, admin, appservice, globals, key_backups, + manager::Manager, + media, presence, pusher, rooms, sending, service::{Args, Map, Service}, transaction_ids, uiaa, users, }; @@ -30,19 +27,12 @@ pub struct Services { pub media: Arc, pub sending: Arc, - manager: Mutex>>>, - workers: Mutex, + manager: Mutex>>, pub(crate) service: Map, pub server: Arc, pub db: Arc, } -type Workers = JoinSet; -type WorkerResult = (Arc, Result<()>); -type WorkersLocked<'a> = MutexGuard<'a, Workers>; - -const RESTART_DELAY_MS: u64 = 2500; - impl Services { pub fn build(server: Arc, db: Arc) -> Result { let mut service: Map = BTreeMap::new(); @@ -94,7 +84,6 @@ impl Services { sending: build!(sending::Service), globals: build!(globals::Service), manager: Mutex::new(None), - workers: Mutex::new(JoinSet::new()), service, server, db, @@ -108,13 +97,13 @@ impl Services { globals::migrations::migrations(&self.db, &self.globals.config).await?; globals::emerg_access::init_emergency_access(); - let mut workers = self.workers.lock().await; - for service in self.service.values() { - self.start_worker(&mut workers, service).await?; - } - - debug!("Starting service manager..."); - self.manager_start().await?; + self.manager + .lock() + .await + .insert(Manager::new(self)) + .clone() + .start() + .await?; if self.globals.allow_check_for_updates() { let handle = globals::updates::start_check_for_updates_task(); @@ -135,16 +124,16 @@ impl Services { _ = updates_handle.await; } - debug!("Stopping service manager..."); - self.manager_stop().await; + if let Some(manager) = self.manager.lock().await.as_ref() { + manager.stop().await; + } debug_info!("Services shutdown complete."); } pub async fn poll(&self) -> Result<()> { - if let Some(manager) = self.manager.lock().await.deref_mut() { - trace!("Polling service manager..."); - return manager.await?; + if let Some(manager) = self.manager.lock().await.as_ref() { + return manager.poll().await; } Ok(()) @@ -190,111 +179,4 @@ impl Services { service.interrupt(); } } - - async fn manager_start(&self) -> Result<()> { - debug!("Starting service manager..."); - self.manager.lock().await.get_or_insert_with(|| { - self.server - .runtime() - .spawn(async move { crate::services().manager().await }) - }); - - Ok(()) - } - - async fn manager_stop(&self) { - if let Some(manager) = self.manager.lock().await.take() { - debug!("Waiting for service manager..."); - if let Err(e) = manager.await { - error!("Manager shutdown error: {e:?}"); - } - } - } - - async fn manager(&self) -> Result<()> { - loop { - let mut workers = self.workers.lock().await; - tokio::select! { - result = workers.join_next() => match result { - Some(Ok(result)) => self.handle_result(&mut workers, result).await?, - Some(Err(error)) => self.handle_abort(&mut workers, Error::from(error)).await?, - None => break, - } - } - } - - debug!("Worker manager finished"); - Ok(()) - } - - async fn handle_abort(&self, _workers: &mut WorkersLocked<'_>, error: Error) -> Result<()> { - // not supported until service can be associated with abort - unimplemented!("unexpected worker task abort {error:?}"); - } - - async fn handle_result(&self, workers: &mut WorkersLocked<'_>, result: WorkerResult) -> Result<()> { - let (service, result) = result; - match result { - Ok(()) => self.handle_finished(workers, &service).await, - Err(error) => self.handle_error(workers, &service, error).await, - } - } - - async fn handle_finished(&self, _workers: &mut WorkersLocked<'_>, service: &Arc) -> Result<()> { - debug!("service {:?} worker finished", service.name()); - Ok(()) - } - - async fn handle_error( - &self, workers: &mut WorkersLocked<'_>, service: &Arc, error: Error, - ) -> Result<()> { - let name = service.name(); - error!("service {name:?} aborted: {error}"); - - if !self.server.running() { - debug_warn!("service {name:?} error ignored on shutdown."); - return Ok(()); - } - - if !error.is_panic() { - return Err(error); - } - - let delay = Duration::from_millis(RESTART_DELAY_MS); - warn!("service {name:?} worker restarting after {} delay", time::pretty(delay)); - sleep(delay).await; - - self.start_worker(workers, service).await - } - - /// Start the worker in a task for the service. - async fn start_worker(&self, workers: &mut WorkersLocked<'_>, service: &Arc) -> Result<()> { - if !self.server.running() { - return Err(Error::Err(format!( - "Service {:?} worker not starting during server shutdown.", - service.name() - ))); - } - - debug!("Service {:?} worker starting...", service.name()); - workers.spawn_on(worker(service.clone()), self.server.runtime()); - - Ok(()) - } -} - -/// Base frame for service worker. This runs in a tokio::task. All errors and -/// panics from the worker are caught and returned cleanly. The JoinHandle -/// should never error with a panic, and if so it should propagate, but it may -/// error with an Abort which the manager should handle along with results to -/// determine if the worker should be restarted. -async fn worker(service: Arc) -> WorkerResult { - let service_ = Arc::clone(&service); - let result = AssertUnwindSafe(service_.worker()) - .catch_unwind() - .await - .map_err(Error::from_panic); - - // flattens JoinError for panic into worker's Error - (service, result.unwrap_or_else(Err)) } From aa7a3102001b2b9609db7a96d10232d63b8ee24f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 13 Jul 2024 07:05:51 +0000 Subject: [PATCH 116/158] move media startup into service Signed-off-by: Jason Volk --- src/service/media/mod.rs | 8 ++++++++ src/service/services.rs | 1 - 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index 3cb8fda8..9b1cbe06 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -3,6 +3,7 @@ mod tests; use std::{collections::HashMap, io::Cursor, num::Saturating as Sat, path::PathBuf, sync::Arc, time::SystemTime}; +use async_trait::async_trait; use base64::{engine::general_purpose, Engine as _}; use conduit::{checked, debug, debug_error, error, utils, Error, Result, Server}; use data::Data; @@ -47,6 +48,7 @@ pub struct Service { pub url_preview_mutex: RwLock>>>, } +#[async_trait] impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { @@ -56,6 +58,12 @@ impl crate::Service for Service { })) } + async fn worker(self: Arc) -> Result<()> { + self.create_media_dir().await?; + + Ok(()) + } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } diff --git a/src/service/services.rs b/src/service/services.rs index 62e73fd1..88b58299 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -93,7 +93,6 @@ impl Services { pub(super) async fn start(&self) -> Result<()> { debug_info!("Starting services..."); - self.media.create_media_dir().await?; globals::migrations::migrations(&self.db, &self.globals.config).await?; globals::emerg_access::init_emergency_access(); From 084751ae381fe6f304f5335251ed1ef4b254b064 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 13 Jul 2024 08:07:49 +0000 Subject: [PATCH 117/158] refactor globals::updates into a Service Signed-off-by: Jason Volk --- src/admin/query/globals.rs | 2 +- src/service/globals/data.rs | 22 +------ src/service/globals/mod.rs | 21 +++---- src/service/globals/updates.rs | 76 ---------------------- src/service/mod.rs | 1 + src/service/services.rs | 24 +++---- src/service/updates/mod.rs | 112 +++++++++++++++++++++++++++++++++ 7 files changed, 133 insertions(+), 125 deletions(-) delete mode 100644 src/service/globals/updates.rs create mode 100644 src/service/updates/mod.rs diff --git a/src/admin/query/globals.rs b/src/admin/query/globals.rs index 2e22d688..9bdd38fc 100644 --- a/src/admin/query/globals.rs +++ b/src/admin/query/globals.rs @@ -26,7 +26,7 @@ pub(super) async fn globals(subcommand: Globals) -> Result { let timer = tokio::time::Instant::now(); - let results = services().globals.db.last_check_for_updates_id(); + let results = services().updates.last_check_for_updates_id(); let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index 254a3d9c..281c2a94 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -14,9 +14,6 @@ use ruma::{ use crate::services; -const COUNTER: &[u8] = b"c"; -const LAST_CHECK_FOR_UPDATES_COUNT: &[u8] = b"u"; - pub struct Data { global: Arc, todeviceid_events: Arc, @@ -35,6 +32,8 @@ pub struct Data { counter: RwLock, } +const COUNTER: &[u8] = b"c"; + impl Data { pub(super) fn new(db: &Arc) -> Self { Self { @@ -93,23 +92,6 @@ impl Data { .map_or(Ok(0_u64), utils::u64_from_bytes) } - pub fn last_check_for_updates_id(&self) -> Result { - self.global - .get(LAST_CHECK_FOR_UPDATES_COUNT)? - .map_or(Ok(0_u64), |bytes| { - utils::u64_from_bytes(&bytes) - .map_err(|_| Error::bad_database("last check for updates count has invalid bytes.")) - }) - } - - #[inline] - pub fn update_check_for_updates_id(&self, id: u64) -> Result<()> { - self.global - .insert(LAST_CHECK_FOR_UPDATES_COUNT, &id.to_be_bytes())?; - - Ok(()) - } - #[tracing::instrument(skip(self), level = "debug")] pub async fn watch(&self, user_id: &UserId, device_id: &DeviceId) -> Result<()> { let userid_bytes = user_id.as_bytes().to_vec(); diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 16830d87..0a0d0d8e 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -1,9 +1,8 @@ mod client; mod data; -pub(super) mod emerg_access; +mod emerg_access; pub(super) mod migrations; pub(crate) mod resolver; -pub(super) mod updates; use std::{ collections::{BTreeMap, HashMap}, @@ -12,6 +11,7 @@ use std::{ time::Instant, }; +use async_trait::async_trait; use conduit::{error, trace, Config, Result}; use data::Data; use ipaddress::IPAddress; @@ -22,7 +22,7 @@ use ruma::{ DeviceId, OwnedEventId, OwnedRoomAliasId, OwnedServerName, OwnedServerSigningKeyId, OwnedUserId, RoomAliasId, RoomVersionId, ServerName, UserId, }; -use tokio::{sync::Mutex, task::JoinHandle}; +use tokio::sync::Mutex; use url::Url; use crate::services; @@ -41,7 +41,6 @@ pub struct Service { pub bad_event_ratelimiter: Arc>>, pub bad_signature_ratelimiter: Arc, RateLimitState>>>, pub bad_query_ratelimiter: Arc>>, - pub updates_handle: Mutex>>, pub stateres_mutex: Arc>, pub server_user: OwnedUserId, pub admin_alias: OwnedRoomAliasId, @@ -49,6 +48,7 @@ pub struct Service { type RateLimitState = (Instant, u32); // Time if last failed try, number of failed tries +#[async_trait] impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let config = &args.server.config; @@ -103,7 +103,6 @@ impl crate::Service for Service { bad_event_ratelimiter: Arc::new(RwLock::new(HashMap::new())), bad_signature_ratelimiter: Arc::new(RwLock::new(HashMap::new())), bad_query_ratelimiter: Arc::new(RwLock::new(HashMap::new())), - updates_handle: Mutex::new(None), stateres_mutex: Arc::new(Mutex::new(())), admin_alias: RoomAliasId::parse(format!("#admins:{}", &config.server_name)) .expect("#admins:server_name is valid alias name"), @@ -122,6 +121,12 @@ impl crate::Service for Service { Ok(Arc::new(s)) } + async fn worker(self: Arc) -> Result<()> { + emerg_access::init_emergency_access(); + + Ok(()) + } + fn memory_usage(&self, out: &mut dyn Write) -> Result<()> { self.resolver.memory_usage(out)?; @@ -181,12 +186,6 @@ impl Service { #[inline] pub fn current_count(&self) -> Result { Ok(self.db.current_count()) } - #[tracing::instrument(skip(self), level = "debug")] - pub fn last_check_for_updates_id(&self) -> Result { self.db.last_check_for_updates_id() } - - #[tracing::instrument(skip(self), level = "debug")] - pub fn update_check_for_updates_id(&self, id: u64) -> Result<()> { self.db.update_check_for_updates_id(id) } - pub async fn watch(&self, user_id: &UserId, device_id: &DeviceId) -> Result<()> { self.db.watch(user_id, device_id).await } diff --git a/src/service/globals/updates.rs b/src/service/globals/updates.rs deleted file mode 100644 index c6ac9fff..00000000 --- a/src/service/globals/updates.rs +++ /dev/null @@ -1,76 +0,0 @@ -use std::time::Duration; - -use ruma::events::room::message::RoomMessageEventContent; -use serde::Deserialize; -use tokio::{task::JoinHandle, time::interval}; -use tracing::{error, warn}; - -use crate::{ - conduit::{Error, Result}, - services, -}; - -const CHECK_FOR_UPDATES_URL: &str = "https://pupbrain.dev/check-for-updates/stable"; -const CHECK_FOR_UPDATES_INTERVAL: u64 = 7200; // 2 hours - -#[derive(Deserialize)] -struct CheckForUpdatesResponseEntry { - id: u64, - date: String, - message: String, -} -#[derive(Deserialize)] -struct CheckForUpdatesResponse { - updates: Vec, -} - -#[tracing::instrument] -pub fn start_check_for_updates_task() -> JoinHandle<()> { - let timer_interval = Duration::from_secs(CHECK_FOR_UPDATES_INTERVAL); - - services().server.runtime().spawn(async move { - let mut i = interval(timer_interval); - - loop { - i.tick().await; - - if let Err(e) = try_handle_updates().await { - warn!(%e, "Failed to check for updates"); - } - } - }) -} - -#[tracing::instrument(skip_all)] -async fn try_handle_updates() -> Result<()> { - let response = services() - .globals - .client - .default - .get(CHECK_FOR_UPDATES_URL) - .send() - .await?; - - let response = serde_json::from_str::(&response.text().await?) - .map_err(|e| Error::Err(format!("Bad check for updates response: {e}")))?; - - let mut last_update_id = services().globals.last_check_for_updates_id()?; - for update in response.updates { - last_update_id = last_update_id.max(update.id); - if update.id > services().globals.last_check_for_updates_id()? { - error!("{}", update.message); - services() - .admin - .send_message(RoomMessageEventContent::text_plain(format!( - "@room: the following is a message from the conduwuit puppy. it was sent on '{}':\n\n{}", - update.date, update.message - ))) - .await; - } - } - services() - .globals - .update_check_for_updates_id(last_update_id)?; - - Ok(()) -} diff --git a/src/service/mod.rs b/src/service/mod.rs index ba68fae2..6f2f4ee5 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -16,6 +16,7 @@ pub mod rooms; pub mod sending; pub mod transaction_ids; pub mod uiaa; +pub mod updates; pub mod users; extern crate conduit_core as conduit; diff --git a/src/service/services.rs b/src/service/services.rs index 88b58299..cc9ec290 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -9,7 +9,7 @@ use crate::{ manager::Manager, media, presence, pusher, rooms, sending, service::{Args, Map, Service}, - transaction_ids, uiaa, users, + transaction_ids, uiaa, updates, users, }; pub struct Services { @@ -22,10 +22,11 @@ pub struct Services { pub account_data: Arc, pub presence: Arc, pub admin: Arc, - pub globals: Arc, pub key_backups: Arc, pub media: Arc, pub sending: Arc, + pub updates: Arc, + pub globals: Arc, manager: Mutex>>, pub(crate) service: Map, @@ -82,6 +83,7 @@ impl Services { key_backups: build!(key_backups::Service), media: build!(media::Service), sending: build!(sending::Service), + updates: build!(updates::Service), globals: build!(globals::Service), manager: Mutex::new(None), service, @@ -93,9 +95,7 @@ impl Services { pub(super) async fn start(&self) -> Result<()> { debug_info!("Starting services..."); - globals::migrations::migrations(&self.db, &self.globals.config).await?; - globals::emerg_access::init_emergency_access(); - + globals::migrations::migrations(&self.db, &self.server.config).await?; self.manager .lock() .await @@ -104,25 +104,14 @@ impl Services { .start() .await?; - if self.globals.allow_check_for_updates() { - let handle = globals::updates::start_check_for_updates_task(); - _ = self.globals.updates_handle.lock().await.insert(handle); - } - debug_info!("Services startup complete."); Ok(()) } pub(super) async fn stop(&self) { info!("Shutting down services..."); + self.interrupt(); - - debug!("Waiting for update worker..."); - if let Some(updates_handle) = self.globals.updates_handle.lock().await.take() { - updates_handle.abort(); - _ = updates_handle.await; - } - if let Some(manager) = self.manager.lock().await.as_ref() { manager.stop().await; } @@ -173,6 +162,7 @@ impl Services { fn interrupt(&self) { debug!("Interrupting services..."); + for (name, service) in &self.service { trace!("Interrupting {name}"); service.interrupt(); diff --git a/src/service/updates/mod.rs b/src/service/updates/mod.rs new file mode 100644 index 00000000..a7088aba --- /dev/null +++ b/src/service/updates/mod.rs @@ -0,0 +1,112 @@ +use std::{sync::Arc, time::Duration}; + +use async_trait::async_trait; +use conduit::{err, info, utils, warn, Error, Result}; +use database::Map; +use ruma::events::room::message::RoomMessageEventContent; +use serde::Deserialize; +use tokio::{sync::Notify, time::interval}; + +use crate::services; + +pub struct Service { + db: Arc, + interrupt: Notify, + interval: Duration, +} + +#[derive(Deserialize)] +struct CheckForUpdatesResponse { + updates: Vec, +} + +#[derive(Deserialize)] +struct CheckForUpdatesResponseEntry { + id: u64, + date: String, + message: String, +} + +const CHECK_FOR_UPDATES_URL: &str = "https://pupbrain.dev/check-for-updates/stable"; +const CHECK_FOR_UPDATES_INTERVAL: u64 = 7200; // 2 hours +const LAST_CHECK_FOR_UPDATES_COUNT: &[u8] = b"u"; + +#[async_trait] +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: args.db["global"].clone(), + interrupt: Notify::new(), + interval: Duration::from_secs(CHECK_FOR_UPDATES_INTERVAL), + })) + } + + async fn worker(self: Arc) -> Result<()> { + let mut i = interval(self.interval); + loop { + tokio::select! { + () = self.interrupt.notified() => return Ok(()), + _ = i.tick() => (), + } + + if let Err(e) = self.handle_updates().await { + warn!(%e, "Failed to check for updates"); + } + } + } + + fn interrupt(&self) { self.interrupt.notify_waiters(); } + + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { + #[tracing::instrument(skip_all)] + async fn handle_updates(&self) -> Result<()> { + let response = services() + .globals + .client + .default + .get(CHECK_FOR_UPDATES_URL) + .send() + .await?; + + let response = serde_json::from_str::(&response.text().await?) + .map_err(|e| Error::Err(format!("Bad check for updates response: {e}")))?; + + let mut last_update_id = self.last_check_for_updates_id()?; + for update in response.updates { + last_update_id = last_update_id.max(update.id); + if update.id > self.last_check_for_updates_id()? { + info!("{:#}", update.message); + services() + .admin + .send_message(RoomMessageEventContent::text_markdown(format!( + "### the following is a message from the conduwuit puppy\n\nit was sent on `{}`:\n\n@room: {}", + update.date, update.message + ))) + .await; + } + } + self.update_check_for_updates_id(last_update_id)?; + + Ok(()) + } + + #[inline] + pub fn update_check_for_updates_id(&self, id: u64) -> Result<()> { + self.db + .insert(LAST_CHECK_FOR_UPDATES_COUNT, &id.to_be_bytes())?; + + Ok(()) + } + + pub fn last_check_for_updates_id(&self) -> Result { + self.db + .get(LAST_CHECK_FOR_UPDATES_COUNT)? + .map_or(Ok(0_u64), |bytes| { + utils::u64_from_bytes(&bytes) + .map_err(|_| Error::bad_database("last check for updates count has invalid bytes.")) + }) + } +} From b3f2288d07d3bbb730ca3a81ae8763bb7e622984 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 13 Jul 2024 21:02:43 +0000 Subject: [PATCH 118/158] add constant-expression string utils Signed-off-by: Jason Volk --- Cargo.lock | 13 +++++++++++++ Cargo.toml | 3 +++ src/admin/Cargo.toml | 1 + src/api/Cargo.toml | 1 + src/core/Cargo.toml | 1 + src/core/utils/string.rs | 24 ++++++++++++++++++++++++ src/database/Cargo.toml | 1 + src/main/Cargo.toml | 1 + src/router/Cargo.toml | 1 + src/service/Cargo.toml | 1 + 10 files changed, 47 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 02da0b6c..f2036284 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -589,6 +589,7 @@ dependencies = [ "conduit_router", "conduit_service", "console-subscriber", + "const-str", "hardened_malloc-rs", "log", "opentelemetry", @@ -613,6 +614,7 @@ dependencies = [ "conduit_api", "conduit_core", "conduit_service", + "const-str", "futures-util", "log", "ruma", @@ -635,6 +637,7 @@ dependencies = [ "conduit_core", "conduit_database", "conduit_service", + "const-str", "futures-util", "hmac", "http 1.1.0", @@ -665,6 +668,7 @@ dependencies = [ "bytes", "checked_ops", "chrono", + "const-str", "either", "figment", "hardened_malloc-rs", @@ -701,6 +705,7 @@ name = "conduit_database" version = "0.4.5" dependencies = [ "conduit_core", + "const-str", "log", "rust-rocksdb-uwu", "tokio", @@ -720,6 +725,7 @@ dependencies = [ "conduit_api", "conduit_core", "conduit_service", + "const-str", "http 1.1.0", "http-body-util", "hyper 1.4.1", @@ -746,6 +752,7 @@ dependencies = [ "bytes", "conduit_core", "conduit_database", + "const-str", "cyborgtime", "futures-util", "hickory-resolver", @@ -817,6 +824,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-str" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3618cccc083bb987a415d85c02ca6c9994ea5b44731ec28b9ecf09658655fba9" + [[package]] name = "const_panic" version = "0.2.8" diff --git a/Cargo.toml b/Cargo.toml index fcbe53d5..33e2b9d9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,9 @@ version = "0.4.5" [workspace.metadata.crane] name = "conduit" +[workspace.dependencies.const-str] +version = "0.5.7" + [workspace.dependencies.sanitize-filename] version = "0.5.0" diff --git a/src/admin/Cargo.toml b/src/admin/Cargo.toml index 7a092237..1e13fb7a 100644 --- a/src/admin/Cargo.toml +++ b/src/admin/Cargo.toml @@ -30,6 +30,7 @@ clap.workspace = true conduit-api.workspace = true conduit-core.workspace = true conduit-service.workspace = true +const-str.workspace = true futures-util.workspace = true log.workspace = true ruma.workspace = true diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index 45cae73d..356adc1f 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -41,6 +41,7 @@ bytes.workspace = true conduit-core.workspace = true conduit-database.workspace = true conduit-service.workspace = true +const-str.workspace = true futures-util.workspace = true hmac.workspace = true http.workspace = true diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index e47f673e..73ee0152 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -55,6 +55,7 @@ axum.workspace = true bytes.workspace = true checked_ops.workspace = true chrono.workspace = true +const-str.workspace = true either.workspace = true figment.workspace = true http-body-util.workspace = true diff --git a/src/core/utils/string.rs b/src/core/utils/string.rs index ec423d53..106d0cb7 100644 --- a/src/core/utils/string.rs +++ b/src/core/utils/string.rs @@ -2,6 +2,30 @@ use crate::Result; pub const EMPTY: &str = ""; +/// Constant expression to bypass format! if the argument is a string literal +/// but not a format string. If the literal is a format string then String is +/// returned otherwise the input (i.e. &'static str) is returned. If multiple +/// arguments are provided the first is assumed to be a format string. +#[macro_export] +macro_rules! format_maybe { + ($s:literal) => { + if $crate::is_format!($s) { std::format!($s).into() } else { $s.into() } + }; + + ($($args:expr),*) => { + std::format!($($args),*).into() + }; +} + +/// Constant expression to decide if a literal is a format string. Note: could +/// use some improvement. +#[macro_export] +macro_rules! is_format { + ($s:literal) => { + ::const_str::contains!($s, "{") && ::const_str::contains!($s, "}") + }; +} + /// Find the common prefix from a collection of strings and return a slice /// ``` /// use conduit_core::utils::string::common_prefix; diff --git a/src/database/Cargo.toml b/src/database/Cargo.toml index 8b0d3fc3..34d98416 100644 --- a/src/database/Cargo.toml +++ b/src/database/Cargo.toml @@ -36,6 +36,7 @@ zstd_compression = [ [dependencies] conduit-core.workspace = true +const-str.workspace = true log.workspace = true rust-rocksdb.workspace = true tokio.workspace = true diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index fa0e5874..8dc2a34d 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -147,6 +147,7 @@ log.workspace = true tracing.workspace = true tracing-subscriber.workspace = true clap.workspace = true +const-str.workspace = true opentelemetry.workspace = true opentelemetry.optional = true diff --git a/src/router/Cargo.toml b/src/router/Cargo.toml index 5312984a..38e6adc7 100644 --- a/src/router/Cargo.toml +++ b/src/router/Cargo.toml @@ -55,6 +55,7 @@ conduit-admin.workspace = true conduit-api.workspace = true conduit-core.workspace = true conduit-service.workspace = true +const-str.workspace = true log.workspace = true tokio.workspace = true tower.workspace = true diff --git a/src/service/Cargo.toml b/src/service/Cargo.toml index f59a6036..d2c9785f 100644 --- a/src/service/Cargo.toml +++ b/src/service/Cargo.toml @@ -42,6 +42,7 @@ base64.workspace = true bytes.workspace = true conduit-core.workspace = true conduit-database.workspace = true +const-str.workspace = true cyborgtime.workspace = true futures-util.workspace = true hickory-resolver.workspace = true From 05efd9b04471fc6e2dd9f373bc8ef505d65cf572 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 13 Jul 2024 21:11:05 +0000 Subject: [PATCH 119/158] elaborate error macro and apply at various callsites Signed-off-by: Jason Volk --- src/admin/server/commands.rs | 11 +- src/admin/utils.rs | 10 +- src/api/client/directory.rs | 7 +- src/api/client/sync.rs | 5 +- src/core/error/err.rs | 103 +++++++++++++----- src/core/error/mod.rs | 23 ++-- src/core/server.rs | 14 +-- src/core/utils/hash/argon.rs | 4 +- src/database/util.rs | 7 +- src/router/serve/unix.rs | 9 +- src/service/appservice/mod.rs | 4 +- src/service/manager.rs | 7 +- src/service/mod.rs | 2 +- src/service/rooms/alias/mod.rs | 2 +- src/service/rooms/auth_chain/mod.rs | 11 +- src/service/rooms/event_handler/mod.rs | 9 +- .../rooms/event_handler/parse_incoming_pdu.rs | 9 +- src/service/rooms/spaces/mod.rs | 10 +- src/service/rooms/state_accessor/mod.rs | 18 +-- src/service/rooms/state_cache/mod.rs | 20 ++-- src/service/sending/mod.rs | 4 +- src/service/sending/resolve.rs | 10 +- src/service/updates/mod.rs | 2 +- 23 files changed, 161 insertions(+), 140 deletions(-) diff --git a/src/admin/server/commands.rs b/src/admin/server/commands.rs index 229f05e1..e4503736 100644 --- a/src/admin/server/commands.rs +++ b/src/admin/server/commands.rs @@ -1,4 +1,4 @@ -use conduit::{utils::time, warn, Error, Result}; +use conduit::{utils::time, warn, Err, Result}; use ruma::events::room::message::RoomMessageEventContent; use crate::services; @@ -88,11 +88,10 @@ pub(super) async fn restart(_body: Vec<&str>, force: bool) -> Result (OwnedRoomId, u64, String) { /// Parses user ID pub(crate) fn parse_user_id(user_id: &str) -> Result { UserId::parse_with_server_name(user_id.to_lowercase(), services().globals.server_name()) - .map_err(|e| Error::Err(format!("The supplied username is not a valid username: {e}"))) + .map_err(|e| err!("The supplied username is not a valid username: {e}")) } /// Parses user ID as our local user @@ -41,7 +41,7 @@ pub(crate) fn parse_local_user_id(user_id: &str) -> Result { let user_id = parse_user_id(user_id)?; if !user_is_local(&user_id) { - return Err(Error::Err(String::from("User does not belong to our server."))); + return Err!("User {user_id:?} does not belong to our server."); } Ok(user_id) @@ -52,11 +52,11 @@ pub(crate) fn parse_active_local_user_id(user_id: &str) -> Result { let user_id = parse_local_user_id(user_id)?; if !services().users.exists(&user_id)? { - return Err(Error::Err(String::from("User does not exist on this server."))); + return Err!("User {user_id:?} does not exist on this server."); } if services().users.is_deactivated(&user_id)? { - return Err(Error::Err(String::from("User is deactivated."))); + return Err!("User {user_id:?} is deactivated."); } Ok(user_id) diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index 7d2aff0d..68bd0dff 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -1,4 +1,5 @@ use axum_client_ip::InsecureClientIp; +use conduit::{err, info, warn, Error, Result}; use ruma::{ api::{ client::{ @@ -18,9 +19,8 @@ use ruma::{ }, uint, RoomId, ServerName, UInt, UserId, }; -use tracing::{error, info, warn}; -use crate::{service::server_is_ours, services, Error, Result, Ruma}; +use crate::{service::server_is_ours, services, Ruma}; /// # `POST /_matrix/client/v3/publicRooms` /// @@ -271,8 +271,7 @@ pub(crate) async fn get_public_rooms_filtered_helper( _ => None, }) .map_err(|e| { - error!("Invalid room join rule event in database: {}", e); - Error::BadDatabase("Invalid room join rule event in database.") + err!(Database(error!("Invalid room join rule event in database: {e}"))) }) }) .transpose()? diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index dd1f3401..e425616b 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -8,7 +8,7 @@ use std::{ use conduit::{ error, utils::math::{ruma_from_u64, ruma_from_usize, usize_from_ruma, usize_from_u64_truncated}, - PduCount, + Err, PduCount, }; use ruma::{ api::client::{ @@ -545,8 +545,7 @@ async fn load_joined_room( // Database queries: let Some(current_shortstatehash) = services().rooms.state.get_room_shortstatehash(room_id)? else { - error!("Room {} has no state", room_id); - return Err(Error::BadDatabase("Room has no state")); + return Err!(Database(error!("Room {room_id} has no state"))); }; let since_shortstatehash = services() diff --git a/src/core/error/err.rs b/src/core/error/err.rs index 12dd4f3b..ea596644 100644 --- a/src/core/error/err.rs +++ b/src/core/error/err.rs @@ -1,3 +1,36 @@ +//! Error construction macros +//! +//! These are specialized macros specific to this project's patterns for +//! throwing Errors; they make Error construction succinct and reduce clutter. +//! They are developed from folding existing patterns into the macro while +//! fixing several anti-patterns in the codebase. +//! +//! - The primary macros `Err!` and `err!` are provided. `Err!` simply wraps +//! `err!` in the Result variant to reduce `Err(err!(...))` boilerplate, thus +//! `err!` can be used in any case. +//! +//! 1. The macro makes the general Error construction easy: `return +//! Err!("something went wrong")` replaces the prior `return +//! Err(Error::Err("something went wrong".to_owned()))`. +//! +//! 2. The macro integrates format strings automatically: `return +//! Err!("something bad: {msg}")` replaces the prior `return +//! Err(Error::Err(format!("something bad: {msg}")))`. +//! +//! 3. The macro scopes variants of Error: `return Err!(Database("problem with +//! bad database."))` replaces the prior `return Err(Error::Database("problem +//! with bad database."))`. +//! +//! 4. The macro matches and scopes some special-case sub-variants, for example +//! with ruma ErrorKind: `return Err!(Request(MissingToken("you must provide +//! an access token")))`. +//! +//! 5. The macro fixes the anti-pattern of repeating messages in an error! log +//! and then again in an Error construction, often slightly different due to +//! the Error variant not supporting a format string. Instead `return +//! Err(Database(error!("problem with db: {msg}")))` logs the error at the +//! callsite and then returns the error with the same string. Caller has the +//! option of replacing `error!` with `debug_error!`. #[macro_export] macro_rules! Err { ($($args:tt)*) => { @@ -7,36 +40,56 @@ macro_rules! Err { #[macro_export] macro_rules! err { - (error!($($args:tt),+)) => {{ - $crate::error!($($args),+); - $crate::error::Error::Err(std::format!($($args),+)) + (Config($item:literal, $($args:expr),*)) => {{ + $crate::error!(config = %$item, $($args),*); + $crate::error::Error::Config($item, $crate::format_maybe!($($args),*)) }}; - (debug_error!($($args:tt),+)) => {{ - $crate::debug_error!($($args),+); - $crate::error::Error::Err(std::format!($($args),+)) + (Request(Forbidden($level:ident!($($args:expr),*)))) => {{ + $crate::$level!($($args),*); + $crate::error::Error::Request( + ::ruma::api::client::error::ErrorKind::forbidden(), + $crate::format_maybe!($($args),*) + ) }}; - ($variant:ident(error!($($args:tt),+))) => {{ - $crate::error!($($args),+); - $crate::error::Error::$variant(std::format!($($args),+)) - }}; - - ($variant:ident(debug_error!($($args:tt),+))) => {{ - $crate::debug_error!($($args),+); - $crate::error::Error::$variant(std::format!($($args),+)) - }}; - - (Config($item:literal, $($args:tt),+)) => {{ - $crate::error!(config = %$item, $($args),+); - $crate::error::Error::Config($item, std::format!($($args),+)) - }}; - - ($variant:ident($($args:tt),+)) => { - $crate::error::Error::$variant(std::format!($($args),+)) + (Request(Forbidden($($args:expr),*))) => { + $crate::error::Error::Request( + ::ruma::api::client::error::ErrorKind::forbidden(), + $crate::format_maybe!($($args),*) + ) }; - ($string:literal$(,)? $($args:tt),*) => { - $crate::error::Error::Err(std::format!($string, $($args),*)) + (Request($variant:ident($level:ident!($($args:expr),*)))) => {{ + $crate::$level!($($args),*); + $crate::error::Error::Request( + ::ruma::api::client::error::ErrorKind::$variant, + $crate::format_maybe!($($args),*) + ) + }}; + + (Request($variant:ident($($args:expr),*))) => { + $crate::error::Error::Request( + ::ruma::api::client::error::ErrorKind::$variant, + $crate::format_maybe!($($args),*) + ) + }; + + ($variant:ident($level:ident!($($args:expr),*))) => {{ + $crate::$level!($($args),*); + $crate::error::Error::$variant($crate::format_maybe!($($args),*)) + }}; + + ($variant:ident($($args:expr),*)) => { + $crate::error::Error::$variant($crate::format_maybe!($($args),*)) + }; + + ($level:ident!($($args:expr),*)) => {{ + $crate::$level!($($args),*); + $crate::error::Error::Err($crate::format_maybe!($($args),*)) + }}; + + ($($args:expr),*) => { + $crate::error::Error::Err($crate::format_maybe!($($args),*)) }; } diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index cbc22f3e..3adfde97 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -3,7 +3,7 @@ mod log; mod panic; mod response; -use std::{any::Any, convert::Infallible, fmt}; +use std::{any::Any, borrow::Cow, convert::Infallible, fmt}; pub use log::*; @@ -64,7 +64,9 @@ pub enum Error { #[error("{0}")] Mxid(#[from] ruma::IdParseError), #[error("{0}: {1}")] - BadRequest(ruma::api::client::error::ErrorKind, &'static str), + BadRequest(ruma::api::client::error::ErrorKind, &'static str), //TODO: remove + #[error("{0}: {1}")] + Request(ruma::api::client::error::ErrorKind, Cow<'static, str>), #[error("from {0}: {1}")] Redaction(ruma::OwnedServerName, ruma::canonical_json::RedactionError), #[error("Remote server {0} responded with: {1}")] @@ -76,11 +78,9 @@ pub enum Error { #[error("Arithmetic operation failed: {0}")] Arithmetic(&'static str), #[error("There was a problem with the '{0}' directive in your configuration: {1}")] - Config(&'static str, String), + Config(&'static str, Cow<'static, str>), #[error("{0}")] - BadDatabase(&'static str), - #[error("{0}")] - Database(String), + Database(Cow<'static, str>), #[error("{0}")] BadServerResponse(&'static str), #[error("{0}")] @@ -88,14 +88,11 @@ pub enum Error { // unique / untyped #[error("{0}")] - Err(String), + Err(Cow<'static, str>), } impl Error { - pub fn bad_database(message: &'static str) -> Self { - error!("BadDatabase: {}", message); - Self::BadDatabase(message) - } + pub fn bad_database(message: &'static str) -> Self { crate::err!(Database(error!("{message}"))) } /// Sanitizes public-facing errors that can leak sensitive information. pub fn sanitized_string(&self) -> String { @@ -121,7 +118,7 @@ impl Error { match self { Self::Federation(_, error) => response::ruma_error_kind(error).clone(), - Self::BadRequest(kind, _) => kind.clone(), + Self::BadRequest(kind, _) | Self::Request(kind, _) => kind.clone(), _ => Unknown, } } @@ -129,7 +126,7 @@ impl Error { pub fn status_code(&self) -> http::StatusCode { match self { Self::Federation(_, ref error) | Self::RumaError(ref error) => error.status_code, - Self::BadRequest(ref kind, _) => response::bad_request_code(kind), + Self::BadRequest(ref kind, _) | Self::Request(ref kind, _) => response::bad_request_code(kind), Self::Conflict(_) => http::StatusCode::CONFLICT, _ => http::StatusCode::INTERNAL_SERVER_ERROR, } diff --git a/src/core/server.rs b/src/core/server.rs index 575924d3..752680ad 100644 --- a/src/core/server.rs +++ b/src/core/server.rs @@ -5,7 +5,7 @@ use std::{ use tokio::{runtime, sync::broadcast}; -use crate::{config::Config, log, Error, Result}; +use crate::{config::Config, log, Err, Result}; /// Server runtime state; public portion pub struct Server { @@ -65,15 +65,15 @@ impl Server { pub fn reload(&self) -> Result<()> { if cfg!(not(conduit_mods)) { - return Err(Error::Err("Reloading not enabled".into())); + return Err!("Reloading not enabled"); } if self.reloading.swap(true, Ordering::AcqRel) { - return Err(Error::Err("Reloading already in progress".into())); + return Err!("Reloading already in progress"); } if self.stopping.swap(true, Ordering::AcqRel) { - return Err(Error::Err("Shutdown already in progress".into())); + return Err!("Shutdown already in progress"); } self.signal("SIGINT").inspect_err(|_| { @@ -84,7 +84,7 @@ impl Server { pub fn restart(&self) -> Result<()> { if self.restarting.swap(true, Ordering::AcqRel) { - return Err(Error::Err("Restart already in progress".into())); + return Err!("Restart already in progress"); } self.shutdown() @@ -93,7 +93,7 @@ impl Server { pub fn shutdown(&self) -> Result<()> { if self.stopping.swap(true, Ordering::AcqRel) { - return Err(Error::Err("Shutdown already in progress".into())); + return Err!("Shutdown already in progress"); } self.signal("SIGTERM") @@ -102,7 +102,7 @@ impl Server { pub fn signal(&self, sig: &'static str) -> Result<()> { if let Err(e) = self.signal.send(sig) { - return Err(Error::Err(format!("Failed to send signal: {e}"))); + return Err!("Failed to send signal: {e}"); } Ok(()) diff --git a/src/core/utils/hash/argon.rs b/src/core/utils/hash/argon.rs index 98cef00e..0a1e1e14 100644 --- a/src/core/utils/hash/argon.rs +++ b/src/core/utils/hash/argon.rs @@ -5,7 +5,7 @@ use argon2::{ PasswordVerifier, Version, }; -use crate::{Error, Result}; +use crate::{err, Error, Result}; const M_COST: u32 = Params::DEFAULT_M_COST; // memory size in 1 KiB blocks const T_COST: u32 = Params::DEFAULT_T_COST; // nr of iterations @@ -44,7 +44,7 @@ pub(super) fn verify_password(password: &str, password_hash: &str) -> Result<()> .map_err(map_err) } -fn map_err(e: password_hash::Error) -> Error { Error::Err(e.to_string()) } +fn map_err(e: password_hash::Error) -> Error { err!("{e}") } #[cfg(test)] mod tests { diff --git a/src/database/util.rs b/src/database/util.rs index 513cedc8..f0ccbcbe 100644 --- a/src/database/util.rs +++ b/src/database/util.rs @@ -1,4 +1,4 @@ -use conduit::Result; +use conduit::{err, Result}; #[inline] pub(crate) fn result(r: std::result::Result) -> Result { @@ -10,4 +10,7 @@ pub(crate) fn and_then(t: T) -> Result { Ok(t) } pub(crate) fn or_else(e: rocksdb::Error) -> Result { Err(map_err(e)) } -pub(crate) fn map_err(e: rocksdb::Error) -> conduit::Error { conduit::Error::Database(e.into_string()) } +pub(crate) fn map_err(e: rocksdb::Error) -> conduit::Error { + let string = e.into_string(); + err!(Database(error!("{string}"))) +} diff --git a/src/router/serve/unix.rs b/src/router/serve/unix.rs index e876b2ac..266936dd 100644 --- a/src/router/serve/unix.rs +++ b/src/router/serve/unix.rs @@ -10,7 +10,7 @@ use axum::{ extract::{connect_info::IntoMakeServiceWithConnectInfo, Request}, Router, }; -use conduit::{debug_error, error::infallible, trace, Error, Result, Server}; +use conduit::{debug, debug_error, error::infallible, info, trace, warn, Err, Result, Server}; use hyper::{body::Incoming, service::service_fn}; use hyper_util::{ rt::{TokioExecutor, TokioIo}, @@ -23,7 +23,6 @@ use tokio::{ task::JoinSet, }; use tower::{Service, ServiceExt}; -use tracing::{debug, info, warn}; type MakeService = IntoMakeServiceWithConnectInfo; @@ -97,19 +96,19 @@ async fn init(server: &Arc) -> Result { let dir = path.parent().unwrap_or_else(|| Path::new("/")); if let Err(e) = fs::create_dir_all(dir).await { - return Err(Error::Err(format!("Failed to create {dir:?} for socket {path:?}: {e}"))); + return Err!("Failed to create {dir:?} for socket {path:?}: {e}"); } let listener = UnixListener::bind(path); if let Err(e) = listener { - return Err(Error::Err(format!("Failed to bind listener {path:?}: {e}"))); + return Err!("Failed to bind listener {path:?}: {e}"); } let socket_perms = config.unix_socket_perms.to_string(); let octal_perms = u32::from_str_radix(&socket_perms, 8).expect("failed to convert octal permissions"); let perms = std::fs::Permissions::from_mode(octal_perms); if let Err(e) = fs::set_permissions(&path, perms).await { - return Err(Error::Err(format!("Failed to set socket {path:?} permissions: {e}"))); + return Err!("Failed to set socket {path:?} permissions: {e}"); } info!("Listening at {:?}", path); diff --git a/src/service/appservice/mod.rs b/src/service/appservice/mod.rs index f7cf0b6b..24c9b8b0 100644 --- a/src/service/appservice/mod.rs +++ b/src/service/appservice/mod.rs @@ -2,7 +2,7 @@ mod data; use std::{collections::BTreeMap, sync::Arc}; -use conduit::Result; +use conduit::{err, Result}; use data::Data; use futures_util::Future; use regex::RegexSet; @@ -171,7 +171,7 @@ impl Service { .write() .await .remove(service_name) - .ok_or_else(|| crate::Error::Err("Appservice not found".to_owned()))?; + .ok_or(err!("Appservice not found"))?; // remove the appservice from the database self.db.unregister_appservice(service_name)?; diff --git a/src/service/manager.rs b/src/service/manager.rs index 84a28a76..af59b4a4 100644 --- a/src/service/manager.rs +++ b/src/service/manager.rs @@ -1,6 +1,6 @@ use std::{panic::AssertUnwindSafe, sync::Arc, time::Duration}; -use conduit::{debug, debug_warn, error, trace, utils::time, warn, Error, Result, Server}; +use conduit::{debug, debug_warn, error, trace, utils::time, warn, Err, Error, Result, Server}; use futures_util::FutureExt; use tokio::{ sync::{Mutex, MutexGuard}, @@ -129,10 +129,7 @@ impl Manager { /// Start the worker in a task for the service. async fn start_worker(&self, workers: &mut WorkersLocked<'_>, service: &Arc) -> Result<()> { if !self.server.running() { - return Err(Error::Err(format!( - "Service {:?} worker not starting during server shutdown.", - service.name() - ))); + return Err!("Service {:?} worker not starting during server shutdown.", service.name()); } debug!("Service {:?} worker starting...", service.name()); diff --git a/src/service/mod.rs b/src/service/mod.rs index 6f2f4ee5..81e0be3b 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -24,7 +24,7 @@ extern crate conduit_database as database; use std::sync::{Arc, RwLock}; -pub(crate) use conduit::{config, debug_error, debug_info, debug_warn, utils, Config, Error, Result, Server}; +pub(crate) use conduit::{config, debug_error, debug_warn, utils, Config, Error, Result, Server}; pub use conduit::{pdu, PduBuilder, PduCount, PduEvent}; use database::Database; pub(crate) use service::{Args, Service}; diff --git a/src/service/rooms/alias/mod.rs b/src/service/rooms/alias/mod.rs index 97eb0f48..792f5c98 100644 --- a/src/service/rooms/alias/mod.rs +++ b/src/service/rooms/alias/mod.rs @@ -171,7 +171,7 @@ impl Service { .rooms .alias .resolve_local_alias(room_alias)? - .ok_or_else(|| err!(BadConfig("Room does not exist.")))?, + .ok_or_else(|| err!(Request(NotFound("Room does not exist."))))?, )); } } diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index 2919ea7e..4e8c7bb2 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -5,9 +5,9 @@ use std::{ sync::Arc, }; -use conduit::{debug, error, trace, validated, warn, Error, Result}; +use conduit::{debug, error, trace, validated, warn, Err, Result}; use data::Data; -use ruma::{api::client::error::ErrorKind, EventId, RoomId}; +use ruma::{EventId, RoomId}; use crate::services; @@ -143,8 +143,11 @@ impl Service { match services().rooms.timeline.get_pdu(&event_id) { Ok(Some(pdu)) => { if pdu.room_id != room_id { - error!(?event_id, ?pdu, "auth event for incorrect room_id"); - return Err(Error::BadRequest(ErrorKind::forbidden(), "Evil event in db")); + return Err!(Request(Forbidden( + "auth event {event_id:?} for incorrect room {} which is not {}", + pdu.room_id, + room_id + ))); } for auth_event in &pdu.auth_events { let sauthevent = services() diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 33c00643..6cb23b9f 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -10,7 +10,7 @@ use std::{ }; use conduit::{ - debug, debug_error, debug_info, error, info, trace, + debug, debug_error, debug_info, err, error, info, trace, utils::{math::continue_exponential_backoff_secs, MutexMap}, warn, Error, Result, }; @@ -1382,11 +1382,8 @@ impl Service { } fn get_room_version_id(create_event: &PduEvent) -> Result { - let create_event_content: RoomCreateEventContent = - serde_json::from_str(create_event.content.get()).map_err(|e| { - error!("Invalid create event: {}", e); - Error::BadDatabase("Invalid create event in db") - })?; + let create_event_content: RoomCreateEventContent = serde_json::from_str(create_event.content.get()) + .map_err(|e| err!(Database("Invalid create event: {e}")))?; Ok(create_event_content.room_version) } diff --git a/src/service/rooms/event_handler/parse_incoming_pdu.rs b/src/service/rooms/event_handler/parse_incoming_pdu.rs index 4c907e51..8fcd8549 100644 --- a/src/service/rooms/event_handler/parse_incoming_pdu.rs +++ b/src/service/rooms/event_handler/parse_incoming_pdu.rs @@ -1,4 +1,4 @@ -use conduit::{Error, Result}; +use conduit::{Err, Error, Result}; use ruma::{api::client::error::ErrorKind, CanonicalJsonObject, OwnedEventId, OwnedRoomId, RoomId}; use serde_json::value::RawValue as RawJsonValue; use tracing::warn; @@ -17,15 +17,12 @@ pub fn parse_incoming_pdu(pdu: &RawJsonValue) -> Result<(OwnedEventId, Canonical .ok_or(Error::BadRequest(ErrorKind::InvalidParam, "Invalid room id in pdu"))?; let Ok(room_version_id) = services().rooms.state.get_room_version(&room_id) else { - return Err(Error::Err(format!("Server is not in room {room_id}"))); + return Err!("Server is not in room {room_id}"); }; let Ok((event_id, value)) = gen_event_id_canonical_json(pdu, &room_version_id) else { // Event could not be converted to canonical json - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Could not convert event to canonical json.", - )); + return Err!(Request(InvalidParam("Could not convert event to canonical json."))); }; Ok((event_id, value, room_id)) diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index 03d0d43f..18133fc1 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -7,7 +7,7 @@ use std::{ sync::Arc, }; -use conduit::{checked, debug_info, utils::math::usize_from_f64}; +use conduit::{checked, debug, debug_info, err, utils::math::usize_from_f64, warn, Error, Result}; use lru_cache::LruCache; use ruma::{ api::{ @@ -27,9 +27,8 @@ use ruma::{ OwnedRoomId, OwnedServerName, RoomId, ServerName, UInt, UserId, }; use tokio::sync::Mutex; -use tracing::{debug, error, warn}; -use crate::{services, Error, Result}; +use crate::services; pub struct CachedSpaceHierarchySummary { summary: SpaceHierarchyParentSummary, @@ -380,10 +379,7 @@ impl Service { .map(|s| { serde_json::from_str(s.content.get()) .map(|c: RoomJoinRulesEventContent| c.join_rule) - .map_err(|e| { - error!("Invalid room join rule event in database: {}", e); - Error::BadDatabase("Invalid room join rule event in database.") - }) + .map_err(|e| err!(Database(error!("Invalid room join rule event in database: {e}")))) }) .transpose()? .unwrap_or(JoinRule::Invite); diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 389f12c3..7abe5e0f 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -6,7 +6,7 @@ use std::{ sync::{Arc, Mutex as StdMutex, Mutex}, }; -use conduit::{error, utils::math::usize_from_f64, warn, Error, Result}; +use conduit::{err, error, utils::math::usize_from_f64, warn, Error, Result}; use data::Data; use lru_cache::LruCache; use ruma::{ @@ -454,10 +454,7 @@ impl Service { .map(|c: RoomJoinRulesEventContent| { (c.join_rule.clone().into(), self.allowed_room_ids(c.join_rule)) }) - .map_err(|e| { - error!("Invalid room join rule event in database: {e}"); - Error::BadDatabase("Invalid room join rule event in database.") - }) + .map_err(|e| err!(Database(error!("Invalid room join rule event in database: {e}")))) }) .transpose()? .unwrap_or((SpaceRoomJoinRule::Invite, vec![]))) @@ -483,10 +480,8 @@ impl Service { Ok(self .room_state_get(room_id, &StateEventType::RoomCreate, "")? .map(|s| { - serde_json::from_str::(s.content.get()).map_err(|e| { - error!("Invalid room create event in database: {e}"); - Error::BadDatabase("Invalid room create event in database.") - }) + serde_json::from_str::(s.content.get()) + .map_err(|e| err!(Database(error!("Invalid room create event in database: {e}")))) }) .transpose()? .and_then(|e| e.room_type)) @@ -499,10 +494,7 @@ impl Service { .map_or(Ok(None), |s| { serde_json::from_str::(s.content.get()) .map(|content| Some(content.algorithm)) - .map_err(|e| { - error!("Invalid room encryption event in database: {e}"); - Error::BadDatabase("Invalid room encryption event in database.") - }) + .map_err(|e| err!(Database(error!("Invalid room encryption event in database: {e}")))) }) } } diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index 30fd7bf4..48215817 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -2,7 +2,7 @@ mod data; use std::sync::Arc; -use conduit::{error, warn, Error, Result}; +use conduit::{err, error, warn, Error, Result}; use data::Data; use itertools::Itertools; use ruma::{ @@ -128,10 +128,8 @@ impl Service { .account_data .get(Some(&predecessor.room_id), user_id, RoomAccountDataEventType::Tag)? .map(|event| { - serde_json::from_str(event.get()).map_err(|e| { - warn!("Invalid account data event in db: {e:?}"); - Error::BadDatabase("Invalid account data event in db.") - }) + serde_json::from_str(event.get()) + .map_err(|e| err!(Database(warn!("Invalid account data event in db: {e:?}")))) }) { services() .account_data @@ -144,10 +142,8 @@ impl Service { .account_data .get(None, user_id, GlobalAccountDataEventType::Direct.to_string().into())? .map(|event| { - serde_json::from_str::(event.get()).map_err(|e| { - warn!("Invalid account data event in db: {e:?}"); - Error::BadDatabase("Invalid account data event in db.") - }) + serde_json::from_str::(event.get()) + .map_err(|e| err!(Database(warn!("Invalid account data event in db: {e:?}")))) }) { let mut direct_event = direct_event?; let mut room_ids_updated = false; @@ -185,10 +181,8 @@ impl Service { .into(), )? .map(|event| { - serde_json::from_str::(event.get()).map_err(|e| { - warn!("Invalid account data event in db: {e:?}"); - Error::BadDatabase("Invalid account data event in db.") - }) + serde_json::from_str::(event.get()) + .map_err(|e| err!(Database(warn!("Invalid account data event in db: {e:?}")))) }) .transpose()? .map_or(false, |ignored| { diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index eb708fcf..88b8b189 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -6,7 +6,7 @@ mod sender; use std::fmt::Debug; -use conduit::{Error, Result}; +use conduit::{err, Result}; pub use resolve::{resolve_actual_dest, CachedDest, CachedOverride, FedDest}; use ruma::{ api::{appservice::Registration, OutgoingRequest}, @@ -224,7 +224,7 @@ impl Service { fn dispatch(&self, msg: Msg) -> Result<()> { debug_assert!(!self.sender.is_full(), "channel full"); debug_assert!(!self.sender.is_closed(), "channel closed"); - self.sender.send(msg).map_err(|e| Error::Err(e.to_string())) + self.sender.send(msg).map_err(|e| err!("{e}")) } } diff --git a/src/service/sending/resolve.rs b/src/service/sending/resolve.rs index 91041a2f..8b6bfc95 100644 --- a/src/service/sending/resolve.rs +++ b/src/service/sending/resolve.rs @@ -5,13 +5,12 @@ use std::{ time::SystemTime, }; -use conduit::Err; +use conduit::{debug, debug_error, debug_info, debug_warn, trace, utils::rand, Err, Error, Result}; use hickory_resolver::{error::ResolveError, lookup::SrvLookup}; use ipaddress::IPAddress; use ruma::{OwnedServerName, ServerName}; -use tracing::{debug, error, trace}; -use crate::{debug_error, debug_info, debug_warn, services, utils::rand, Error, Result}; +use crate::services; /// Wraps either an literal IP address plus port, or a hostname plus complement /// (colon-plus-port if it was specified). @@ -346,10 +345,7 @@ fn handle_resolve_error(e: &ResolveError) -> Result<()> { debug!("{e}"); Ok(()) }, - _ => { - error!("DNS {e}"); - Err(Error::Err(e.to_string())) - }, + _ => Err!(error!("DNS {e}")), } } diff --git a/src/service/updates/mod.rs b/src/service/updates/mod.rs index a7088aba..3fb680d6 100644 --- a/src/service/updates/mod.rs +++ b/src/service/updates/mod.rs @@ -72,7 +72,7 @@ impl Service { .await?; let response = serde_json::from_str::(&response.text().await?) - .map_err(|e| Error::Err(format!("Bad check for updates response: {e}")))?; + .map_err(|e| err!("Bad check for updates response: {e}"))?; let mut last_update_id = self.last_check_for_updates_id()?; for update in response.updates { From d2fb6d04c9cb6583814f86366de389fc02e3cf73 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 14 Jul 2024 01:11:03 +0000 Subject: [PATCH 120/158] cleanup pending transactions before sender worker completes Signed-off-by: Jason Volk --- src/service/sending/sender.rs | 40 +++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index cfd5b4bc..2f542dfe 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -3,12 +3,12 @@ use std::{ collections::{BTreeMap, HashMap, HashSet}, fmt::Debug, sync::Arc, - time::Instant, + time::{Duration, Instant}, }; use async_trait::async_trait; use base64::{engine::general_purpose, Engine as _}; -use conduit::{debug, error, utils::math::continue_exponential_backoff_secs, warn}; +use conduit::{debug, debug_warn, error, trace, utils::math::continue_exponential_backoff_secs, warn}; use federation::transactions::send_transaction_message; use futures_util::{future::BoxFuture, stream::FuturesUnordered, StreamExt}; use ruma::{ @@ -24,7 +24,7 @@ use ruma::{ ServerName, UInt, }; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; -use tokio::sync::Mutex; +use tokio::{sync::Mutex, time::sleep_until}; use super::{appservice, data::Data, send, Destination, Msg, SendingEvent, Service}; use crate::{presence::Presence, services, user_is_local, utils::calculate_hash, Error, Result}; @@ -44,6 +44,7 @@ type CurTransactionStatus = HashMap; const DEQUEUE_LIMIT: usize = 48; const SELECT_EDU_LIMIT: usize = 16; +const CLEANUP_TIMEOUT_MS: u64 = 3500; #[async_trait] impl crate::Service for Service { @@ -65,19 +66,22 @@ impl crate::Service for Service { let mut futures: SendingFutures<'_> = FuturesUnordered::new(); let mut statuses: CurTransactionStatus = CurTransactionStatus::new(); - self.initial_transactions(&futures, &mut statuses); + self.initial_requests(&futures, &mut statuses); loop { debug_assert!(!receiver.is_closed(), "channel error"); tokio::select! { request = receiver.recv_async() => match request { Ok(request) => self.handle_request(request, &futures, &mut statuses), - Err(_) => return Ok(()), + Err(_) => break, }, Some(response) = futures.next() => { - self.handle_response(response, &mut futures, &mut statuses); + self.handle_response(response, &futures, &mut statuses); }, } } + self.finish_responses(&mut futures, &mut statuses).await; + + Ok(()) } fn interrupt(&self) { @@ -91,7 +95,7 @@ impl crate::Service for Service { impl Service { fn handle_response( - &self, response: SendingResult, futures: &mut SendingFutures<'_>, statuses: &mut CurTransactionStatus, + &self, response: SendingResult, futures: &SendingFutures<'_>, statuses: &mut CurTransactionStatus, ) { match response { Ok(dest) => self.handle_response_ok(&dest, futures, statuses), @@ -100,7 +104,7 @@ impl Service { } fn handle_response_err( - dest: Destination, _futures: &mut SendingFutures<'_>, statuses: &mut CurTransactionStatus, e: &Error, + dest: Destination, _futures: &SendingFutures<'_>, statuses: &mut CurTransactionStatus, e: &Error, ) { debug!(dest = ?dest, "{e:?}"); statuses.entry(dest).and_modify(|e| { @@ -151,7 +155,25 @@ impl Service { } } - fn initial_transactions(&self, futures: &SendingFutures<'_>, statuses: &mut CurTransactionStatus) { + async fn finish_responses(&self, futures: &mut SendingFutures<'_>, statuses: &mut CurTransactionStatus) { + let now = Instant::now(); + let timeout = Duration::from_millis(CLEANUP_TIMEOUT_MS); + let deadline = now.checked_add(timeout).unwrap_or(now); + loop { + trace!("Waiting for {} requests to complete...", futures.len()); + tokio::select! { + () = sleep_until(deadline.into()) => break, + response = futures.next() => match response { + Some(response) => self.handle_response(response, futures, statuses), + None => return, + } + } + } + + debug_warn!("Leaving with {} unfinished requests...", futures.len()); + } + + fn initial_requests(&self, futures: &SendingFutures<'_>, statuses: &mut CurTransactionStatus) { let keep = usize::try_from(self.startup_netburst_keep).unwrap_or(usize::MAX); let mut txns = HashMap::>::new(); for (key, dest, event) in self.db.active_requests().filter_map(Result::ok) { From e4dc4a1ba59d09ba037a7160ae0e39c0eeb9cc03 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 14 Jul 2024 06:07:54 +0000 Subject: [PATCH 121/158] fix graceful shutdown on unix socket Signed-off-by: Jason Volk --- src/router/serve/unix.rs | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/router/serve/unix.rs b/src/router/serve/unix.rs index 266936dd..b5938673 100644 --- a/src/router/serve/unix.rs +++ b/src/router/serve/unix.rs @@ -3,7 +3,7 @@ use std::{ net::{self, IpAddr, Ipv4Addr}, path::Path, - sync::Arc, + sync::{atomic::Ordering, Arc}, }; use axum::{ @@ -21,12 +21,14 @@ use tokio::{ net::{unix::SocketAddr, UnixListener, UnixStream}, sync::broadcast::{self}, task::JoinSet, + time::{sleep, Duration}, }; use tower::{Service, ServiceExt}; type MakeService = IntoMakeServiceWithConnectInfo; -static NULL_ADDR: net::SocketAddr = net::SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 0); +const NULL_ADDR: net::SocketAddr = net::SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 0); +const FINI_POLL_INTERVAL: Duration = Duration::from_millis(750); #[tracing::instrument(skip_all)] pub(super) async fn serve(server: &Arc, app: Router, mut shutdown: broadcast::Receiver<()>) -> Result<()> { @@ -47,7 +49,7 @@ pub(super) async fn serve(server: &Arc, app: Router, mut shutdown: broad } } - fini(listener, tasks).await; + fini(server, listener, tasks).await; Ok(()) } @@ -111,15 +113,26 @@ async fn init(server: &Arc) -> Result { return Err!("Failed to set socket {path:?} permissions: {e}"); } - info!("Listening at {:?}", path); + info!("Listening at {path:?}"); Ok(listener.unwrap()) } -async fn fini(listener: UnixListener, mut tasks: JoinSet<()>) { +async fn fini(server: &Arc, listener: UnixListener, mut tasks: JoinSet<()>) { let local = listener.local_addr(); + debug!("Closing listener at {local:?} ..."); drop(listener); + + debug!("Waiting for requests to finish..."); + while server.metrics.requests_spawn_active.load(Ordering::Relaxed) > 0 { + tokio::select! { + _ = tasks.join_next() => {} + () = sleep(FINI_POLL_INTERVAL) => {} + } + } + + debug!("Shutting down..."); tasks.shutdown().await; if let Ok(local) = local { From 5ec49b3f622be21ebb7a68a2f712aab230111097 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 14 Jul 2024 05:11:58 +0000 Subject: [PATCH 122/158] split csp into array; integrate error; cleanup type Signed-off-by: Jason Volk --- src/core/error/mod.rs | 2 ++ src/router/layers.rs | 24 ++++++++++++++++-------- src/router/request.rs | 19 +++++++++---------- 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index 3adfde97..069fe60e 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -53,6 +53,8 @@ pub enum Error { Path(#[from] axum::extract::rejection::PathRejection), #[error("{0}")] Http(#[from] http::Error), + #[error("{0}")] + HttpHeader(#[from] http::header::InvalidHeaderValue), // ruma #[error("{0}")] diff --git a/src/router/layers.rs b/src/router/layers.rs index db664b38..8c2e114b 100644 --- a/src/router/layers.rs +++ b/src/router/layers.rs @@ -1,11 +1,11 @@ -use std::{any::Any, io, sync::Arc, time::Duration}; +use std::{any::Any, sync::Arc, time::Duration}; use axum::{ extract::{DefaultBodyLimit, MatchedPath}, Router, }; use axum_client_ip::SecureClientIpSource; -use conduit::Server; +use conduit::{Result, Server}; use http::{ header::{self, HeaderName}, HeaderValue, Method, StatusCode, @@ -22,11 +22,19 @@ use tracing::Level; use crate::{request, router}; -const CONDUWUIT_CSP: &str = "sandbox; default-src 'none'; font-src 'none'; script-src 'none'; frame-ancestors 'none'; \ - form-action 'none'; base-uri 'none';"; -const CONDUWUIT_PERMISSIONS_POLICY: &str = "interest-cohort=(),browsing-topics=()"; +const CONDUWUIT_CSP: &[&str] = &[ + "sandbox", + "default-src 'none'", + "font-src 'none'", + "script-src 'none'", + "frame-ancestors 'none'", + "form-action 'none'", + "base-uri 'none'", +]; -pub(crate) fn build(server: &Arc) -> io::Result { +const CONDUWUIT_PERMISSIONS_POLICY: &[&str] = &["interest-cohort=()", "browsing-topics=()"]; + +pub(crate) fn build(server: &Arc) -> Result { let layers = ServiceBuilder::new(); #[cfg(feature = "sentry_telemetry")] @@ -65,11 +73,11 @@ pub(crate) fn build(server: &Arc) -> io::Result { )) .layer(SetResponseHeaderLayer::if_not_present( HeaderName::from_static("permissions-policy"), - HeaderValue::from_static(CONDUWUIT_PERMISSIONS_POLICY), + HeaderValue::from_str(&CONDUWUIT_PERMISSIONS_POLICY.join(","))?, )) .layer(SetResponseHeaderLayer::if_not_present( header::CONTENT_SECURITY_POLICY, - HeaderValue::from_static(CONDUWUIT_CSP), + HeaderValue::from_str(&CONDUWUIT_CSP.join("; "))?, )) .layer(cors_layer(server)) .layer(body_limit_layer(server)) diff --git a/src/router/request.rs b/src/router/request.rs index 9256fb9c..851bd168 100644 --- a/src/router/request.rs +++ b/src/router/request.rs @@ -1,6 +1,9 @@ use std::sync::{atomic::Ordering, Arc}; -use axum::{extract::State, response::IntoResponse}; +use axum::{ + extract::State, + response::{IntoResponse, Response}, +}; use conduit::{debug, debug_error, debug_warn, defer, error, trace, Error, Result, Server}; use http::{Method, StatusCode, Uri}; use ruma::api::client::error::{Error as RumaError, ErrorBody, ErrorKind}; @@ -8,7 +11,7 @@ use ruma::api::client::error::{Error as RumaError, ErrorBody, ErrorKind}; #[tracing::instrument(skip_all, level = "debug")] pub(crate) async fn spawn( State(server): State>, req: http::Request, next: axum::middleware::Next, -) -> Result { +) -> Result { if !server.running() { debug_warn!("unavailable pending shutdown"); return Err(StatusCode::SERVICE_UNAVAILABLE); @@ -30,7 +33,7 @@ pub(crate) async fn spawn( #[tracing::instrument(skip_all, name = "handle")] pub(crate) async fn handle( State(server): State>, req: http::Request, next: axum::middleware::Next, -) -> Result { +) -> Result { if !server.running() { debug_warn!( method = %req.method(), @@ -57,9 +60,7 @@ pub(crate) async fn handle( handle_result(&method, &uri, result) } -fn handle_result( - method: &Method, uri: &Uri, result: axum::response::Response, -) -> Result { +fn handle_result(method: &Method, uri: &Uri, result: Response) -> Result { handle_result_log(method, uri, &result); match result.status() { StatusCode::METHOD_NOT_ALLOWED => handle_result_405(method, uri, &result), @@ -67,9 +68,7 @@ fn handle_result( } } -fn handle_result_405( - _method: &Method, _uri: &Uri, result: &axum::response::Response, -) -> Result { +fn handle_result_405(_method: &Method, _uri: &Uri, result: &Response) -> Result { let error = Error::RumaError(RumaError { status_code: result.status(), body: ErrorBody::Standard { @@ -81,7 +80,7 @@ fn handle_result_405( Ok(error.into_response()) } -fn handle_result_log(method: &Method, uri: &Uri, result: &axum::response::Response) { +fn handle_result_log(method: &Method, uri: &Uri, result: &Response) { let status = result.status(); let reason = status.canonical_reason().unwrap_or("Unknown Reason"); let code = status.as_u16(); From cce270d93823859daecb25d4b9682fcaf494943c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 25 Jun 2024 05:05:02 +0000 Subject: [PATCH 123/158] tokio metrics Signed-off-by: Jason Volk --- Cargo.lock | 2 ++ Cargo.toml | 7 ++-- src/admin/debug/commands.rs | 34 ++++++++++++++++++ src/admin/debug/mod.rs | 9 +++++ src/core/Cargo.toml | 1 + src/core/metrics/mod.rs | 72 +++++++++++++++++++++++++++++++++++++ src/core/mod.rs | 1 + src/core/server.rs | 24 +++++-------- src/main/main.rs | 4 +-- src/router/layers.rs | 1 + src/router/request.rs | 14 +++++--- src/router/run.rs | 2 +- src/router/serve/plain.rs | 19 +++++++--- 13 files changed, 157 insertions(+), 33 deletions(-) create mode 100644 src/core/metrics/mod.rs diff --git a/Cargo.lock b/Cargo.lock index f2036284..48d1a91f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -694,6 +694,7 @@ dependencies = [ "tikv-jemalloc-sys", "tikv-jemallocator", "tokio", + "tokio-metrics", "tracing", "tracing-core", "tracing-subscriber", @@ -4058,6 +4059,7 @@ checksum = "eace09241d62c98b7eeb1107d4c5c64ca3bd7da92e8c218c153ab3a78f9be112" dependencies = [ "futures-util", "pin-project-lite", + "tokio", "tokio-stream", ] diff --git a/Cargo.toml b/Cargo.toml index 33e2b9d9..e24d84b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -200,6 +200,9 @@ features = [ "io-util", ] +[workspace.dependencies.tokio-metrics] +version = "0.3.1" + [workspace.dependencies.libloading] version = "0.8.3" @@ -382,10 +385,6 @@ version = "0.5.4" default-features = false features = ["use_std"] -[workspace.dependencies.tokio-metrics] -version = "0.3.1" -default-features = false - [workspace.dependencies.console-subscriber] version = "0.3" diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index a5087467..46f71622 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -653,3 +653,37 @@ pub(super) fn memory_stats() -> RoomMessageEventContent { html_body.expect("string result"), ) } + +#[cfg(tokio_unstable)] +pub(super) async fn runtime_metrics(_body: Vec<&str>) -> Result { + let out = services().server.metrics.runtime_metrics().map_or_else( + || "Runtime metrics are not available.".to_owned(), + |metrics| format!("```rs\n{metrics:#?}\n```"), + ); + + Ok(RoomMessageEventContent::text_markdown(out)) +} + +#[cfg(not(tokio_unstable))] +pub(super) async fn runtime_metrics(_body: Vec<&str>) -> Result { + Ok(RoomMessageEventContent::text_markdown( + "Runtime metrics require building with `tokio_unstable`.", + )) +} + +#[cfg(tokio_unstable)] +pub(super) async fn runtime_interval(_body: Vec<&str>) -> Result { + let out = services().server.metrics.runtime_interval().map_or_else( + || "Runtime metrics are not available.".to_owned(), + |metrics| format!("```rs\n{metrics:#?}\n```"), + ); + + Ok(RoomMessageEventContent::text_markdown(out)) +} + +#[cfg(not(tokio_unstable))] +pub(super) async fn runtime_interval(_body: Vec<&str>) -> Result { + Ok(RoomMessageEventContent::text_markdown( + "Runtime metrics require building with `tokio_unstable`.", + )) +} diff --git a/src/admin/debug/mod.rs b/src/admin/debug/mod.rs index eed3b633..7d6cafa7 100644 --- a/src/admin/debug/mod.rs +++ b/src/admin/debug/mod.rs @@ -160,6 +160,13 @@ pub(super) enum DebugCommand { /// - Print extended memory usage MemoryStats, + /// - Print general tokio runtime metric totals. + RuntimeMetrics, + + /// - Print detailed tokio runtime metrics accumulated since last command + /// invocation. + RuntimeInterval, + /// - Developer test stubs #[command(subcommand)] Tester(TesterCommand), @@ -213,6 +220,8 @@ pub(super) async fn process(command: DebugCommand, body: Vec<&str>) -> Result resolve_true_destination(body, server_name, no_cache).await?, DebugCommand::MemoryStats => memory_stats(), + DebugCommand::RuntimeMetrics => runtime_metrics(body).await?, + DebugCommand::RuntimeInterval => runtime_interval(body).await?, DebugCommand::Tester(command) => tester::process(command, body).await?, }) } diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index 73ee0152..453d7b13 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -82,6 +82,7 @@ tikv-jemalloc-ctl.workspace = true tikv-jemalloc-sys.optional = true tikv-jemalloc-sys.workspace = true tokio.workspace = true +tokio-metrics.workspace = true tracing-core.workspace = true tracing-subscriber.workspace = true tracing.workspace = true diff --git a/src/core/metrics/mod.rs b/src/core/metrics/mod.rs new file mode 100644 index 00000000..3ae139a8 --- /dev/null +++ b/src/core/metrics/mod.rs @@ -0,0 +1,72 @@ +use std::sync::atomic::AtomicU32; + +use tokio::runtime; +use tokio_metrics::TaskMonitor; +#[cfg(tokio_unstable)] +use tokio_metrics::{RuntimeIntervals, RuntimeMonitor}; + +pub struct Metrics { + _runtime: Option, + + runtime_metrics: Option, + + task_monitor: Option, + + #[cfg(tokio_unstable)] + _runtime_monitor: Option, + + #[cfg(tokio_unstable)] + runtime_intervals: std::sync::Mutex>, + + // TODO: move stats + pub requests_spawn_active: AtomicU32, + pub requests_spawn_finished: AtomicU32, + pub requests_handle_active: AtomicU32, + pub requests_handle_finished: AtomicU32, + pub requests_panic: AtomicU32, +} + +impl Metrics { + #[must_use] + pub fn new(runtime: Option) -> Self { + #[cfg(tokio_unstable)] + let runtime_monitor = runtime.as_ref().map(RuntimeMonitor::new); + + #[cfg(tokio_unstable)] + let runtime_intervals = runtime_monitor.as_ref().map(RuntimeMonitor::intervals); + + Self { + _runtime: runtime.clone(), + + runtime_metrics: runtime.as_ref().map(runtime::Handle::metrics), + + task_monitor: runtime.map(|_| TaskMonitor::new()), + + #[cfg(tokio_unstable)] + _runtime_monitor: runtime_monitor, + + #[cfg(tokio_unstable)] + runtime_intervals: std::sync::Mutex::new(runtime_intervals), + + requests_spawn_active: AtomicU32::new(0), + requests_spawn_finished: AtomicU32::new(0), + requests_handle_active: AtomicU32::new(0), + requests_handle_finished: AtomicU32::new(0), + requests_panic: AtomicU32::new(0), + } + } + + #[cfg(tokio_unstable)] + pub fn runtime_interval(&self) -> Option { + self.runtime_intervals + .lock() + .expect("locked") + .as_mut() + .map(Iterator::next) + .expect("next interval") + } + + pub fn task_root(&self) -> Option<&TaskMonitor> { self.task_monitor.as_ref() } + + pub fn runtime_metrics(&self) -> Option<&runtime::RuntimeMetrics> { self.runtime_metrics.as_ref() } +} diff --git a/src/core/mod.rs b/src/core/mod.rs index de8057fa..9716b46e 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -3,6 +3,7 @@ pub mod config; pub mod debug; pub mod error; pub mod log; +pub mod metrics; pub mod mods; pub mod pdu; pub mod server; diff --git a/src/core/server.rs b/src/core/server.rs index 752680ad..bf0ab99d 100644 --- a/src/core/server.rs +++ b/src/core/server.rs @@ -1,11 +1,11 @@ use std::{ - sync::atomic::{AtomicBool, AtomicU32, Ordering}, + sync::atomic::{AtomicBool, Ordering}, time::SystemTime, }; use tokio::{runtime, sync::broadcast}; -use crate::{config::Config, log, Err, Result}; +use crate::{config::Config, log::Log, metrics::Metrics, Err, Result}; /// Server runtime state; public portion pub struct Server { @@ -33,33 +33,25 @@ pub struct Server { pub signal: broadcast::Sender<&'static str>, /// Logging subsystem state - pub log: log::Log, + pub log: Log, - /// TODO: move stats - pub requests_spawn_active: AtomicU32, - pub requests_spawn_finished: AtomicU32, - pub requests_handle_active: AtomicU32, - pub requests_handle_finished: AtomicU32, - pub requests_panic: AtomicU32, + /// Metrics subsystem state + pub metrics: Metrics, } impl Server { #[must_use] - pub fn new(config: Config, runtime: Option, log: log::Log) -> Self { + pub fn new(config: Config, runtime: Option, log: Log) -> Self { Self { config, started: SystemTime::now(), stopping: AtomicBool::new(false), reloading: AtomicBool::new(false), restarting: AtomicBool::new(false), - runtime, + runtime: runtime.clone(), signal: broadcast::channel::<&'static str>(1).0, log, - requests_spawn_active: AtomicU32::new(0), - requests_spawn_finished: AtomicU32::new(0), - requests_handle_active: AtomicU32::new(0), - requests_handle_finished: AtomicU32::new(0), - requests_panic: AtomicU32::new(0), + metrics: Metrics::new(runtime), } } diff --git a/src/main/main.rs b/src/main/main.rs index 23f53b4e..959e8610 100644 --- a/src/main/main.rs +++ b/src/main/main.rs @@ -20,7 +20,7 @@ use tokio::runtime; const WORKER_NAME: &str = "conduwuit:worker"; const WORKER_MIN: usize = 2; -const WORKER_KEEPALIVE_MS: u64 = 2500; +const WORKER_KEEPALIVE: u64 = 36; fn main() -> Result<(), Error> { let args = clap::parse(); @@ -29,7 +29,7 @@ fn main() -> Result<(), Error> { .enable_time() .thread_name(WORKER_NAME) .worker_threads(cmp::max(WORKER_MIN, available_parallelism())) - .thread_keep_alive(Duration::from_millis(WORKER_KEEPALIVE_MS)) + .thread_keep_alive(Duration::from_secs(WORKER_KEEPALIVE)) .build() .expect("built runtime"); diff --git a/src/router/layers.rs b/src/router/layers.rs index 8c2e114b..073940f1 100644 --- a/src/router/layers.rs +++ b/src/router/layers.rs @@ -153,6 +153,7 @@ fn body_limit_layer(server: &Server) -> DefaultBodyLimit { DefaultBodyLimit::max fn catch_panic(err: Box) -> http::Response> { conduit_service::services() .server + .metrics .requests_panic .fetch_add(1, std::sync::atomic::Ordering::Release); diff --git a/src/router/request.rs b/src/router/request.rs index 851bd168..ae739984 100644 --- a/src/router/request.rs +++ b/src/router/request.rs @@ -17,11 +17,14 @@ pub(crate) async fn spawn( return Err(StatusCode::SERVICE_UNAVAILABLE); } - let active = server.requests_spawn_active.fetch_add(1, Ordering::Relaxed); + let active = server + .metrics + .requests_spawn_active + .fetch_add(1, Ordering::Relaxed); trace!(active, "enter"); defer! {{ - let active = server.requests_spawn_active.fetch_sub(1, Ordering::Relaxed); - let finished = server.requests_spawn_finished.fetch_add(1, Ordering::Relaxed); + let active = server.metrics.requests_spawn_active.fetch_sub(1, Ordering::Relaxed); + let finished = server.metrics.requests_spawn_finished.fetch_add(1, Ordering::Relaxed); trace!(active, finished, "leave"); }}; @@ -45,12 +48,13 @@ pub(crate) async fn handle( } let active = server + .metrics .requests_handle_active .fetch_add(1, Ordering::Relaxed); trace!(active, "enter"); defer! {{ - let active = server.requests_handle_active.fetch_sub(1, Ordering::Relaxed); - let finished = server.requests_handle_finished.fetch_add(1, Ordering::Relaxed); + let active = server.metrics.requests_handle_active.fetch_sub(1, Ordering::Relaxed); + let finished = server.metrics.requests_handle_finished.fetch_add(1, Ordering::Relaxed); trace!(active, finished, "leave"); }}; diff --git a/src/router/run.rs b/src/router/run.rs index b3ec2285..91507772 100644 --- a/src/router/run.rs +++ b/src/router/run.rs @@ -108,7 +108,7 @@ async fn handle_shutdown(server: &Arc, tx: &Sender<()>, handle: &axum_se error!("failed sending shutdown transaction to channel: {e}"); } - let pending = server.requests_spawn_active.load(Ordering::Relaxed); + let pending = server.metrics.requests_spawn_active.load(Ordering::Relaxed); if pending > 0 { let timeout = Duration::from_secs(36); trace!(pending, ?timeout, "Notifying for graceful shutdown"); diff --git a/src/router/serve/plain.rs b/src/router/serve/plain.rs index b79d342d..08263353 100644 --- a/src/router/serve/plain.rs +++ b/src/router/serve/plain.rs @@ -21,12 +21,21 @@ pub(super) async fn serve( info!("Listening on {addrs:?}"); while join_set.join_next().await.is_some() {} - let spawn_active = server.requests_spawn_active.load(Ordering::Relaxed); - let handle_active = server.requests_handle_active.load(Ordering::Relaxed); + let spawn_active = server.metrics.requests_spawn_active.load(Ordering::Relaxed); + let handle_active = server + .metrics + .requests_handle_active + .load(Ordering::Relaxed); debug_info!( - spawn_finished = server.requests_spawn_finished.load(Ordering::Relaxed), - handle_finished = server.requests_handle_finished.load(Ordering::Relaxed), - panics = server.requests_panic.load(Ordering::Relaxed), + spawn_finished = server + .metrics + .requests_spawn_finished + .load(Ordering::Relaxed), + handle_finished = server + .metrics + .requests_handle_finished + .load(Ordering::Relaxed), + panics = server.metrics.requests_panic.load(Ordering::Relaxed), spawn_active, handle_active, "Stopped listening on {addrs:?}", From 95006f7e46f4865955d38950e212bac50076bad2 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 14 Jul 2024 10:55:39 +0000 Subject: [PATCH 124/158] fix unnecessary preprocessing cfgs Signed-off-by: Jason Volk --- src/core/config/check.rs | 58 +++++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/src/core/config/check.rs b/src/core/config/check.rs index 20a61b70..b36b9c5e 100644 --- a/src/core/config/check.rs +++ b/src/core/config/check.rs @@ -1,23 +1,29 @@ use figment::Figment; use super::DEPRECATED_KEYS; -use crate::{debug, error, info, warn, Config, Err, Result}; +use crate::{debug, debug_info, error, info, warn, Config, Err, Result}; #[allow(clippy::cognitive_complexity)] pub fn check(config: &Config) -> Result<()> { - #[cfg(debug_assertions)] - info!("Note: conduwuit was built without optimisations (i.e. debug build)"); + if cfg!(debug_assertions) { + info!("Note: conduwuit was built without optimisations (i.e. debug build)"); + } - #[cfg(all(feature = "rocksdb", not(feature = "sha256_media")))] // prevents catching this in `--all-features` - warn!( - "Note the rocksdb feature was deleted from conduwuit. SQLite support was removed and RocksDB is the only \ - supported backend now. Please update your build script to remove this feature." - ); - #[cfg(all(feature = "sha256_media", not(feature = "rocksdb")))] // prevents catching this in `--all-features` - warn!( - "Note the sha256_media feature was deleted from conduwuit, it is now fully integrated in a \ - forwards-compatible way. Please update your build script to remove this feature." - ); + // prevents catching this in `--all-features` + if cfg!(all(feature = "rocksdb", not(feature = "sha256_media"))) { + warn!( + "Note the rocksdb feature was deleted from conduwuit. SQLite support was removed and RocksDB is the only \ + supported backend now. Please update your build script to remove this feature." + ); + } + + // prevents catching this in `--all-features` + if cfg!(all(feature = "sha256_media", not(feature = "rocksdb"))) { + warn!( + "Note the sha256_media feature was deleted from conduwuit, it is now fully integrated in a \ + forwards-compatible way. Please update your build script to remove this feature." + ); + } warn_deprecated(config); warn_unknown_key(config); @@ -26,28 +32,27 @@ pub fn check(config: &Config) -> Result<()> { return Err!(Config("sentry_endpoint", "Sentry cannot be enabled without an endpoint set")); } - #[cfg(all(feature = "hardened_malloc", feature = "jemalloc"))] - warn!( - "hardened_malloc and jemalloc are both enabled, this causes jemalloc to be used. If using --all-features, \ - this is harmless." - ); + if cfg!(all(feature = "hardened_malloc", feature = "jemalloc")) { + warn!( + "hardened_malloc and jemalloc are both enabled, this causes jemalloc to be used. If using --all-features, \ + this is harmless." + ); + } - #[cfg(not(unix))] - if config.unix_socket_path.is_some() { + if cfg!(not(unix)) && config.unix_socket_path.is_some() { return Err!(Config( "unix_socket_path", - "UNIX socket support is only available on *nix platforms. Please remove \"unix_socket_path\" from your \ - config.", + "UNIX socket support is only available on *nix platforms. Please remove 'unix_socket_path' from your \ + config." )); } - #[cfg(unix)] - if config.unix_socket_path.is_none() { + if cfg!(unix) && config.unix_socket_path.is_none() { config.get_bind_addrs().iter().for_each(|addr| { use std::path::Path; if addr.ip().is_loopback() { - crate::debug_info!("Found loopback listening address {addr}, running checks if we're in a container.",); + debug_info!("Found loopback listening address {addr}, running checks if we're in a container."); if Path::new("/proc/vz").exists() /* Guest */ && !Path::new("/proc/bz").exists() /* Host */ @@ -90,8 +95,7 @@ pub fn check(config: &Config) -> Result<()> { } // yeah, unless the user built a debug build hopefully for local testing only - #[cfg(not(debug_assertions))] - if config.server_name == "your.server.name" { + if cfg!(not(debug_assertions)) && config.server_name == "your.server.name" { return Err!(Config( "server_name", "You must specify a valid server name for production usage of conduwuit." From e53c2fbc5ab0232b4907db0d7cf21f45d362cd7c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 14 Jul 2024 11:30:10 +0000 Subject: [PATCH 125/158] fix tracing flame envfilter defaults; remove preprocessed cfgs Signed-off-by: Jason Volk --- src/core/config/mod.rs | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 16c7b1a8..336144db 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -166,19 +166,14 @@ pub struct Config { #[serde(default)] pub well_known: WellKnownConfig, #[serde(default)] - #[cfg(feature = "perf_measurements")] pub allow_jaeger: bool, #[serde(default = "default_jaeger_filter")] - #[cfg(feature = "perf_measurements")] pub jaeger_filter: String, #[serde(default)] - #[cfg(feature = "perf_measurements")] pub tracing_flame: bool, #[serde(default = "default_tracing_flame_filter")] - #[cfg(feature = "perf_measurements")] pub tracing_flame_filter: String, #[serde(default = "default_tracing_flame_output_path")] - #[cfg(feature = "perf_measurements")] pub tracing_flame_output_path: String, #[serde(default)] pub proxy: ProxyConfig, @@ -672,11 +667,8 @@ impl fmt::Display for Config { } &lst.into_iter().join(", ") }); - #[cfg(feature = "zstd_compression")] line("Zstd HTTP Compression", &self.zstd_compression.to_string()); - #[cfg(feature = "gzip_compression")] line("Gzip HTTP Compression", &self.gzip_compression.to_string()); - #[cfg(feature = "brotli_compression")] line("Brotli HTTP Compression", &self.brotli_compression.to_string()); line("RocksDB database LOG level", &self.rocksdb_log_level); line("RocksDB database LOG to stderr", &self.rocksdb_log_stderr.to_string()); @@ -925,10 +917,13 @@ fn default_pusher_idle_timeout() -> u64 { 15 } fn default_max_fetch_prev_events() -> u16 { 100_u16 } -#[cfg(feature = "perf_measurements")] -fn default_tracing_flame_filter() -> String { "trace,h2=off".to_owned() } +fn default_tracing_flame_filter() -> String { + cfg!(debug_assertions) + .then_some("trace,h2=off") + .unwrap_or("info") + .to_owned() +} -#[cfg(feature = "perf_measurements")] fn default_jaeger_filter() -> String { cfg!(debug_assertions) .then_some("trace,h2=off") @@ -936,7 +931,6 @@ fn default_jaeger_filter() -> String { .to_owned() } -#[cfg(feature = "perf_measurements")] fn default_tracing_flame_output_path() -> String { "./tracing.folded".to_owned() } fn default_trusted_servers() -> Vec { vec![OwnedServerName::try_from("matrix.org").unwrap()] } From 101fdbc9b1a1587e1eef7e95286ef73633b0af83 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 13 Jul 2024 16:14:28 -0400 Subject: [PATCH 126/158] bump cargo.lock Updating bytes v1.6.0 -> v1.6.1 Updating cc v1.1.0 -> v1.1.3 Updating http-body v1.0.0 -> v1.0.1 Updating syn v2.0.70 -> v2.0.71 Updating thiserror v1.0.61 -> v1.0.62 Updating thiserror-impl v1.0.61 -> v1.0.62 Signed-off-by: strawberry Co-authored-by: Jason Volk --- Cargo.lock | 82 +++++++++++++++++++++++++++--------------------------- Cargo.toml | 4 +-- 2 files changed, 43 insertions(+), 43 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 48d1a91f..7f8e3f77 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -118,7 +118,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -129,7 +129,7 @@ checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -192,7 +192,7 @@ dependencies = [ "bytes", "futures-util", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "http-body-util", "hyper 1.4.1", "hyper-util", @@ -253,7 +253,7 @@ dependencies = [ "bytes", "futures-util", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", @@ -276,7 +276,7 @@ dependencies = [ "futures-util", "headers", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", @@ -296,7 +296,7 @@ dependencies = [ "bytes", "futures-util", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "http-body-util", "hyper 1.4.1", "hyper-util", @@ -377,7 +377,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -457,9 +457,9 @@ checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" [[package]] name = "bytes" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +checksum = "a12916984aab3fa6e39d655a33e09c0071eb36d6ab3aea5c2d78551f1df6d952" [[package]] name = "bzip2-sys" @@ -474,9 +474,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.0" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaff6f8ce506b9773fa786672d63fc7a191ffea1be33f72bbd4aeacefca9ffc8" +checksum = "18e2d530f35b40a84124146478cd16f34225306a8441998836466a2e2961c950" dependencies = [ "jobserver", "libc", @@ -562,7 +562,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -1022,7 +1022,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -1126,7 +1126,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -1262,7 +1262,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -1543,7 +1543,7 @@ dependencies = [ "markup5ever", "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -1590,9 +1590,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", "http 1.1.0", @@ -1607,7 +1607,7 @@ dependencies = [ "bytes", "futures-util", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "pin-project-lite", ] @@ -1664,7 +1664,7 @@ dependencies = [ "futures-util", "h2 0.4.5", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "httparse", "httpdate", "itoa", @@ -1715,7 +1715,7 @@ dependencies = [ "futures-channel", "futures-util", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "hyper 1.4.1", "pin-project-lite", "socket2", @@ -1952,7 +1952,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -2461,7 +2461,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -2554,7 +2554,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -2642,7 +2642,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", "version_check", "yansi", ] @@ -2667,7 +2667,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -2857,7 +2857,7 @@ dependencies = [ "h2 0.4.5", "hickory-resolver", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "http-body-util", "hyper 1.4.1", "hyper-rustls", @@ -3076,7 +3076,7 @@ dependencies = [ "quote", "ruma-identifiers-validation 0.9.5 (git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5)", "serde", - "syn 2.0.70", + "syn 2.0.71", "toml", ] @@ -3531,7 +3531,7 @@ checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -3824,9 +3824,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.70" +version = "2.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f0209b68b3613b093e0ec905354eccaedcfe83b8cb37cbdeae64026c3064c16" +checksum = "b146dcf730474b4bcd16c311627b31ede9ab149045db4d6088b3becaea046462" dependencies = [ "proc-macro2", "quote", @@ -3884,22 +3884,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.61" +version = "1.0.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" +checksum = "f2675633b1499176c2dff06b0856a27976a8f9d436737b4cf4f312d4d91d8bbb" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.61" +version = "1.0.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" +checksum = "d20468752b09f49e909e55a5d338caa8bedf615594e9d80bc4c565d30faf798c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -4048,7 +4048,7 @@ checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -4224,7 +4224,7 @@ dependencies = [ "futures-core", "futures-util", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "http-body-util", "pin-project-lite", "tokio", @@ -4265,7 +4265,7 @@ source = "git+https://github.com/girlbossceo/tracing?rev=b348dca742af641c47bc390 dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -4535,7 +4535,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", "wasm-bindgen-shared", ] @@ -4569,7 +4569,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", "wasm-bindgen-backend", "wasm-bindgen-shared", ] diff --git a/Cargo.toml b/Cargo.toml index e24d84b8..f9696252 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,7 +53,7 @@ version = "0.8.5" # Used for the http request / response body type for Ruma endpoints used with reqwest [workspace.dependencies.bytes] -version = "1.6.0" +version = "1.6.1" [workspace.dependencies.http-body-util] version = "0.1.1" @@ -251,7 +251,7 @@ default-features = false # Used for conduit::Error type [workspace.dependencies.thiserror] -version = "1.0.61" +version = "1.0.62" # Used when hashing the state [workspace.dependencies.ring] From 3d73b53136daef69c2f023aaec353e6c1ad1c4a1 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 13 Jul 2024 22:16:48 -0400 Subject: [PATCH 127/158] update complement results we dont implement authenticated media yet Signed-off-by: strawberry --- tests/test_results/complement/test_results.jsonl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_results/complement/test_results.jsonl b/tests/test_results/complement/test_results.jsonl index 2f3db95d..873397ba 100644 --- a/tests/test_results/complement/test_results.jsonl +++ b/tests/test_results/complement/test_results.jsonl @@ -138,7 +138,8 @@ {"Action":"fail","Test":"TestKnockingInMSC3787Room/Knocking_on_a_room_with_join_rule_'knock'_should_succeed#01"} {"Action":"fail","Test":"TestKnockingInMSC3787Room/Users_in_the_room_see_a_user's_membership_update_when_they_knock"} {"Action":"fail","Test":"TestKnockingInMSC3787Room/Users_in_the_room_see_a_user's_membership_update_when_they_knock#01"} -{"Action":"pass","Test":"TestLocalPngThumbnail"} +{"Action":"fail","Test":"TestLocalPngThumbnail"} +{"Action":"fail","Test":"TestLocalPngThumbnail/test_/_matrix/client/v1/media_endpoint"} {"Action":"fail","Test":"TestMediaFilenames"} {"Action":"fail","Test":"TestMediaFilenames/Parallel"} {"Action":"fail","Test":"TestMediaFilenames/Parallel/ASCII"} @@ -181,7 +182,8 @@ {"Action":"pass","Test":"TestOutboundFederationProfile/Outbound_federation_can_query_profile_data"} {"Action":"pass","Test":"TestOutboundFederationSend"} {"Action":"pass","Test":"TestRemoteAliasRequestsUnderstandUnicode"} -{"Action":"pass","Test":"TestRemotePngThumbnail"} +{"Action":"fail","Test":"TestRemotePngThumbnail"} +{"Action":"fail","Test":"TestRemotePngThumbnail/test_/_matrix/client/v1/media_endpoint"} {"Action":"fail","Test":"TestRemotePresence"} {"Action":"fail","Test":"TestRemotePresence/Presence_changes_are_also_reported_to_remote_room_members"} {"Action":"fail","Test":"TestRemotePresence/Presence_changes_to_UNAVAILABLE_are_reported_to_remote_room_members"} From 1ab77aeb9143cd0c3ff6ac750a8bef68b0455250 Mon Sep 17 00:00:00 2001 From: Charles Hall Date: Sat, 13 Jul 2024 15:48:07 -0400 Subject: [PATCH 128/158] don't strip binary for dev builds Signed-off-by: strawberry --- nix/pkgs/main/default.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nix/pkgs/main/default.nix b/nix/pkgs/main/default.nix index e3d23a43..ffb1de96 100644 --- a/nix/pkgs/main/default.nix +++ b/nix/pkgs/main/default.nix @@ -126,6 +126,8 @@ commonAttrs = { ]; }; + dontStrip = profile == "dev"; + buildInputs = lib.optional (featureEnabled "jemalloc") rust-jemalloc-sys'; nativeBuildInputs = [ From fcb6c8a1138cfe58e31c7841c07e5a5a2908935c Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 13 Jul 2024 15:54:14 -0400 Subject: [PATCH 129/158] bump rust-rocksdb Signed-off-by: strawberry --- Cargo.lock | 8 ++++---- deps/rust-rocksdb/Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7f8e3f77..12b96665 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3138,8 +3138,8 @@ dependencies = [ [[package]] name = "rust-librocksdb-sys" -version = "0.23.1+9.3.1" -source = "git+https://github.com/zaidoon1/rust-rocksdb?rev=db1ba33c2b78ad228e2525e8902d059c24fc81a1#db1ba33c2b78ad228e2525e8902d059c24fc81a1" +version = "0.24.0+9.4.0" +source = "git+https://github.com/zaidoon1/rust-rocksdb?rev=4056a3b0f823013fec49f6d0b3e5698856e6476a#4056a3b0f823013fec49f6d0b3e5698856e6476a" dependencies = [ "bindgen", "bzip2-sys", @@ -3155,8 +3155,8 @@ dependencies = [ [[package]] name = "rust-rocksdb" -version = "0.27.1" -source = "git+https://github.com/zaidoon1/rust-rocksdb?rev=db1ba33c2b78ad228e2525e8902d059c24fc81a1#db1ba33c2b78ad228e2525e8902d059c24fc81a1" +version = "0.28.0" +source = "git+https://github.com/zaidoon1/rust-rocksdb?rev=4056a3b0f823013fec49f6d0b3e5698856e6476a#4056a3b0f823013fec49f6d0b3e5698856e6476a" dependencies = [ "libc", "rust-librocksdb-sys", diff --git a/deps/rust-rocksdb/Cargo.toml b/deps/rust-rocksdb/Cargo.toml index 57b71491..cf49e1ad 100644 --- a/deps/rust-rocksdb/Cargo.toml +++ b/deps/rust-rocksdb/Cargo.toml @@ -27,7 +27,7 @@ malloc-usable-size = ["rust-rocksdb/malloc-usable-size"] [dependencies.rust-rocksdb] git = "https://github.com/zaidoon1/rust-rocksdb" -rev = "db1ba33c2b78ad228e2525e8902d059c24fc81a1" +rev = "4056a3b0f823013fec49f6d0b3e5698856e6476a" #branch = "master" default-features = false From 2c0bfac43e1610b215d92d4cb35466a6b0e3d10a Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 13 Jul 2024 16:01:58 -0400 Subject: [PATCH 130/158] nix: bump flake.lock and rocksdb to v9.4.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit • Updated input 'attic': 'github:zhaofengli/attic/717cc95983cdc357bc347d70be20ced21f935843' (2024-06-01) → 'github:zhaofengli/attic/6139576a3ce6bb992e0f6c3022528ec233e45f00' (2024-07-09) • Updated input 'complement': 'github:matrix-org/complement/bc97f1ddc1cd7485faf80c8935ee2641f3e1b57c' (2024-07-02) → 'github:matrix-org/complement/0d14432e010482ea9e13a6f7c47c1533c0c9d62f' (2024-07-10) • Updated input 'crane': 'github:ipetkov/crane/0aed560c5c0a61c9385bddff471a13036203e11c' (2024-07-06) → 'github:ipetkov/crane/2d83156f23c43598cf44e152c33a59d3892f8b29' (2024-07-09) • Updated input 'fenix': 'github:nix-community/fenix/abc0549e3560189462a7d394cc9d50af4608d103' (2024-07-08) → 'github:nix-community/fenix/5087b12a595ee73131a944d922f24d81dae05725' (2024-07-13) • Updated input 'fenix/rust-analyzer-src': 'github:rust-lang/rust-analyzer/a5b21ea0aa644dffd7cf958b43f11f221d53404e' (2024-07-07) → 'github:rust-lang/rust-analyzer/ffbc5ad993d5cd2f3b8bcf9a511165470944ab91' (2024-07-11) • Updated input 'liburing': 'github:axboe/liburing/7b3245583069bd481190c9da18f22e9fc8c3a805' (2024-06-22) → 'github:axboe/liburing/1d674f83b7d0f07553ac44d99a401b05853d9dbe' (2024-07-12) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/655a58a72a6601292512670343087c2d75d859c1' (2024-07-08) → 'github:NixOS/nixpkgs/7e7c39ea35c5cdd002cd4588b03a3fb9ece6fad9' (2024-07-12) • Updated input 'rocksdb': 'github:girlbossceo/rocksdb/a935c0273e1ba44eacf88ce3685a9b9831486155' (2024-07-02) → 'github:girlbossceo/rocksdb/911f4243e69c2e320a7a209bf1f5f3ff5f825495' (2024-07-13) Signed-off-by: strawberry --- flake.lock | 50 +++++++++++++++++++++++++------------------------- flake.nix | 3 +-- 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/flake.lock b/flake.lock index 4e6721b2..9d7a162e 100644 --- a/flake.lock +++ b/flake.lock @@ -9,11 +9,11 @@ "nixpkgs-stable": "nixpkgs-stable" }, "locked": { - "lastModified": 1717279440, - "narHash": "sha256-kH04ReTjxOpQumgWnqy40vvQLSnLGxWP6RF3nq5Esrk=", + "lastModified": 1720542474, + "narHash": "sha256-aKjJ/4l2I9+wNGTaOGRsuS3M1+IoTibqgEMPDikXm04=", "owner": "zhaofengli", "repo": "attic", - "rev": "717cc95983cdc357bc347d70be20ced21f935843", + "rev": "6139576a3ce6bb992e0f6c3022528ec233e45f00", "type": "github" }, "original": { @@ -81,11 +81,11 @@ "complement": { "flake": false, "locked": { - "lastModified": 1719903368, - "narHash": "sha256-PPzgxM4Bir+Zh9FUV/v+RBxEYeJxYVmi/BYo3uqt268=", + "lastModified": 1720637557, + "narHash": "sha256-oZz6nCmFmdJZpC+K1iOG2KkzTI6rlAmndxANPDVU7X0=", "owner": "matrix-org", "repo": "complement", - "rev": "bc97f1ddc1cd7485faf80c8935ee2641f3e1b57c", + "rev": "0d14432e010482ea9e13a6f7c47c1533c0c9d62f", "type": "github" }, "original": { @@ -123,11 +123,11 @@ ] }, "locked": { - "lastModified": 1720226507, - "narHash": "sha256-yHVvNsgrpyNTXZBEokL8uyB2J6gB1wEx0KOJzoeZi1A=", + "lastModified": 1720546058, + "narHash": "sha256-iU2yVaPIZm5vMGdlT0+57vdB/aPq/V5oZFBRwYw+HBM=", "owner": "ipetkov", "repo": "crane", - "rev": "0aed560c5c0a61c9385bddff471a13036203e11c", + "rev": "2d83156f23c43598cf44e152c33a59d3892f8b29", "type": "github" }, "original": { @@ -209,11 +209,11 @@ "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1720420198, - "narHash": "sha256-OIuDb6pHDyGpo7YMFyuRzMLcHm7mRvlYOz0Ht7ps2sU=", + "lastModified": 1720852044, + "narHash": "sha256-3NBYz8VuXuKU+8ONd9NFafCNjPEGHIZQ2Mdoam1a4mY=", "owner": "nix-community", "repo": "fenix", - "rev": "abc0549e3560189462a7d394cc9d50af4608d103", + "rev": "5087b12a595ee73131a944d922f24d81dae05725", "type": "github" }, "original": { @@ -381,11 +381,11 @@ "liburing": { "flake": false, "locked": { - "lastModified": 1719025212, - "narHash": "sha256-kD0yhjNStqC6uFqC1AxBwUpc/HlSFtiKrV+gwDyroDc=", + "lastModified": 1720798442, + "narHash": "sha256-gtPppAoksMLW4GuruQ36nf4EAqIA1Bs6V9Xcx8dBxrQ=", "owner": "axboe", "repo": "liburing", - "rev": "7b3245583069bd481190c9da18f22e9fc8c3a805", + "rev": "1d674f83b7d0f07553ac44d99a401b05853d9dbe", "type": "github" }, "original": { @@ -606,11 +606,11 @@ }, "nixpkgs_4": { "locked": { - "lastModified": 1720418205, - "narHash": "sha256-cPJoFPXU44GlhWg4pUk9oUPqurPlCFZ11ZQPk21GTPU=", + "lastModified": 1720768451, + "narHash": "sha256-EYekUHJE2gxeo2pM/zM9Wlqw1Uw2XTJXOSAO79ksc4Y=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "655a58a72a6601292512670343087c2d75d859c1", + "rev": "7e7c39ea35c5cdd002cd4588b03a3fb9ece6fad9", "type": "github" }, "original": { @@ -673,16 +673,16 @@ "rocksdb": { "flake": false, "locked": { - "lastModified": 1719949653, - "narHash": "sha256-DYx7XHH2GEh17GukKhXs6laM6l+eugCmRkF0adpi9wk=", + "lastModified": 1720900786, + "narHash": "sha256-Vta9Um/RRuWwZ46BjXftV06iWLm/j/9MX39emXUvSAY=", "owner": "girlbossceo", "repo": "rocksdb", - "rev": "a935c0273e1ba44eacf88ce3685a9b9831486155", + "rev": "911f4243e69c2e320a7a209bf1f5f3ff5f825495", "type": "github" }, "original": { "owner": "girlbossceo", - "ref": "v9.3.1", + "ref": "v9.4.0", "repo": "rocksdb", "type": "github" } @@ -705,11 +705,11 @@ "rust-analyzer-src": { "flake": false, "locked": { - "lastModified": 1720344064, - "narHash": "sha256-STmaV9Zu74QtkGGrbr9uMhskwagfCjJqOAYapXabiuk=", + "lastModified": 1720717809, + "narHash": "sha256-6I+fm+nTLF/iaj7ffiFGlSY7POmubwUaPA/Wq0Bm53M=", "owner": "rust-lang", "repo": "rust-analyzer", - "rev": "a5b21ea0aa644dffd7cf958b43f11f221d53404e", + "rev": "ffbc5ad993d5cd2f3b8bcf9a511165470944ab91", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index e20da52b..36ebc997 100644 --- a/flake.nix +++ b/flake.nix @@ -9,8 +9,7 @@ flake-utils.url = "github:numtide/flake-utils?ref=main"; nix-filter.url = "github:numtide/nix-filter?ref=main"; nixpkgs.url = "github:NixOS/nixpkgs?ref=nixos-unstable"; - # https://github.com/girlbossceo/rocksdb/commit/db6df0b185774778457dabfcbd822cb81760cade - rocksdb = { url = "github:girlbossceo/rocksdb?ref=v9.3.1"; flake = false; }; + rocksdb = { url = "github:girlbossceo/rocksdb?ref=v9.4.0"; flake = false; }; liburing = { url = "github:axboe/liburing?ref=master"; flake = false; }; }; From 7009f56a7ae6dae52da692c0bd8bee76d5aa95f5 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 13 Jul 2024 16:02:44 -0400 Subject: [PATCH 131/158] fix user directory publishing access controls Signed-off-by: strawberry --- src/api/client/directory.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index 68bd0dff..8e12c034 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -117,7 +117,12 @@ pub(crate) async fn set_room_visibility_route( return Err(Error::BadRequest(ErrorKind::NotFound, "Room not found")); } - user_can_publish_room(sender_user, &body.room_id)?; + if !user_can_publish_room(sender_user, &body.room_id)? { + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "User is not allowed to publish this room", + )); + } match &body.visibility { room::Visibility::Public => { @@ -377,8 +382,8 @@ fn user_can_publish_room(user_id: &UserId, room_id: &RoomId) -> Result { Ok(event.sender == user_id) } else { return Err(Error::BadRequest( - ErrorKind::Unauthorized, - "You are not allowed to publish this room to the room directory", + ErrorKind::forbidden(), + "User is not allowed to publish this room", )); } } From a1bfd7a018afe2135da01ecc0ddd7ce88c444316 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 13 Jul 2024 16:09:24 -0400 Subject: [PATCH 132/158] nix: dont include `experimental` feature on all-features builds this is just future infra for it Signed-off-by: strawberry --- flake.nix | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/flake.nix b/flake.nix index 36ebc997..0024e176 100644 --- a/flake.nix +++ b/flake.nix @@ -122,8 +122,12 @@ default = scopeHost.main; all-features = scopeHost.main.override { all_features = true; - # this is non-functional on nix for some reason - disable_features = ["hardened_malloc"]; + disable_features = [ + # this is non-functional on nix for some reason + "hardened_malloc" + # dont include experimental features + "experimental" + ]; }; hmalloc = scopeHost.main.override { features = ["hardened_malloc"]; }; @@ -131,8 +135,12 @@ oci-image-all-features = scopeHost.oci-image.override { main = scopeHost.main.override { all_features = true; - # this is non-functional on nix for some reason - disable_features = ["hardened_malloc"]; + disable_features = [ + # this is non-functional on nix for some reason + "hardened_malloc" + # dont include experimental features + "experimental" + ]; }; }; oci-image-hmalloc = scopeHost.oci-image.override { @@ -174,8 +182,12 @@ name = "${binaryName}-all-features"; value = scopeCrossStatic.main.override { all_features = true; - # this is non-functional on nix for some reason - disable_features = ["hardened_malloc"]; + disable_features = [ + # this is non-functional on nix for some reason + "hardened_malloc" + # dont include experimental features + "experimental" + ]; }; } @@ -199,8 +211,12 @@ value = scopeCrossStatic.oci-image.override { main = scopeCrossStatic.main.override { all_features = true; - # this is non-functional on nix for some reason - disable_features = ["hardened_malloc"]; + disable_features = [ + # this is non-functional on nix for some reason + "hardened_malloc" + # dont include experimental features + "experimental" + ]; }; }; } @@ -228,8 +244,12 @@ (scopeHostStatic.overrideScope (final: prev: { main = prev.main.override { all_features = true; - # this is non-functional on nix for some reason - disable_features = ["hardened_malloc"]; + disable_features = [ + # this is non-functional on nix for some reason + "hardened_malloc" + # dont include experimental features + "experimental" + ]; }; })); devShells.no-features = mkDevShell From bacffd617427d835d0792b87ef64cba8aad95bae Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 13 Jul 2024 16:46:13 -0400 Subject: [PATCH 133/158] rename `conduit_cache_capacity_modifier` to `cache_capacity_modifier` this prefix causes you to require setting the environment variable to `CONDUWUIT_CONDUIT_CACHE_CAPACITY_MODIFIER` alias this so we dont break any configs Signed-off-by: strawberry --- conduwuit-example.toml | 5 ++++- src/core/config/mod.rs | 11 ++++++----- src/database/opts.rs | 2 +- src/service/rooms/auth_chain/data.rs | 2 +- src/service/rooms/spaces/mod.rs | 2 +- src/service/rooms/state_accessor/mod.rs | 4 ++-- src/service/rooms/state_compressor/mod.rs | 2 +- 7 files changed, 16 insertions(+), 12 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index ea095bfa..57093128 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -421,8 +421,11 @@ allow_profile_lookup_federation_requests = true # Set this to any float value to multiply conduwuit's in-memory LRU caches with. # May be useful if you have significant memory to spare to increase performance. +# +# This was previously called `conduit_cache_capacity_modifier` +# # Defaults to 1.0. -#conduit_cache_capacity_modifier = 1.0 +#cache_capacity_modifier = 1.0 # Set this to any float value in megabytes for conduwuit to tell the database engine that this much memory is available for database-related caches. # May be useful if you have significant memory to spare to increase performance. diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 336144db..b40ebb65 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -58,8 +58,8 @@ pub struct Config { #[serde(default = "default_pdu_cache_capacity")] pub pdu_cache_capacity: u32, - #[serde(default = "default_conduit_cache_capacity_modifier")] - pub conduit_cache_capacity_modifier: f64, + #[serde(default = "default_cache_capacity_modifier", alias = "conduit_cache_capacity_modifier")] + pub cache_capacity_modifier: f64, #[serde(default = "default_auth_chain_cache_capacity")] pub auth_chain_cache_capacity: u32, #[serde(default = "default_shorteventid_cache_capacity")] @@ -391,8 +391,9 @@ struct ListeningAddr { addrs: Either>, } -const DEPRECATED_KEYS: &[&str] = &[ +const DEPRECATED_KEYS: &[&str; 9] = &[ "cache_capacity", + "conduit_cache_capacity_modifier", "max_concurrent_requests", "well_known_client", "well_known_server", @@ -484,7 +485,7 @@ impl fmt::Display for Config { ); line("Database backups to keep", &self.database_backups_to_keep.to_string()); line("Database cache capacity (MB)", &self.db_cache_capacity_mb.to_string()); - line("Cache capacity modifier", &self.conduit_cache_capacity_modifier.to_string()); + line("Cache capacity modifier", &self.cache_capacity_modifier.to_string()); line("PDU cache capacity", &self.pdu_cache_capacity.to_string()); line("Auth chain cache capacity", &self.auth_chain_cache_capacity.to_string()); line("Short eventid cache capacity", &self.shorteventid_cache_capacity.to_string()); @@ -847,7 +848,7 @@ fn default_db_cache_capacity_mb() -> f64 { 256.0 } fn default_pdu_cache_capacity() -> u32 { 150_000 } -fn default_conduit_cache_capacity_modifier() -> f64 { 1.0 } +fn default_cache_capacity_modifier() -> f64 { 1.0 } fn default_auth_chain_cache_capacity() -> u32 { 100_000 } diff --git a/src/database/opts.rs b/src/database/opts.rs index 1b9c163f..d2236454 100644 --- a/src/database/opts.rs +++ b/src/database/opts.rs @@ -310,7 +310,7 @@ fn set_table_with_shared_cache( } fn cache_size(config: &Config, base_size: u32, entity_size: usize) -> usize { - let ents = f64::from(base_size) * config.conduit_cache_capacity_modifier; + let ents = f64::from(base_size) * config.cache_capacity_modifier; #[allow(clippy::as_conversions, clippy::cast_sign_loss, clippy::cast_possible_truncation)] (ents as usize) diff --git a/src/service/rooms/auth_chain/data.rs b/src/service/rooms/auth_chain/data.rs index 5efb36c2..4e468234 100644 --- a/src/service/rooms/auth_chain/data.rs +++ b/src/service/rooms/auth_chain/data.rs @@ -16,7 +16,7 @@ impl Data { pub(super) fn new(server: &Arc, db: &Arc) -> Self { let config = &server.config; let cache_size = f64::from(config.auth_chain_cache_capacity); - let cache_size = usize_from_f64(cache_size * config.conduit_cache_capacity_modifier).expect("valid cache size"); + let cache_size = usize_from_f64(cache_size * config.cache_capacity_modifier).expect("valid cache size"); Self { shorteventid_authchain: db["shorteventid_authchain"].clone(), auth_chain_cache: Mutex::new(LruCache::new(cache_size)), diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index 18133fc1..02db7fab 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -161,7 +161,7 @@ impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let config = &args.server.config; let cache_size = f64::from(config.roomid_spacehierarchy_cache_capacity); - let cache_size = cache_size * config.conduit_cache_capacity_modifier; + let cache_size = cache_size * config.cache_capacity_modifier; Ok(Arc::new(Self { roomid_spacehierarchy_cache: Mutex::new(LruCache::new(usize_from_f64(cache_size)?)), })) diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 7abe5e0f..bd3eb0a1 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -45,9 +45,9 @@ impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let config = &args.server.config; let server_visibility_cache_capacity = - f64::from(config.server_visibility_cache_capacity) * config.conduit_cache_capacity_modifier; + f64::from(config.server_visibility_cache_capacity) * config.cache_capacity_modifier; let user_visibility_cache_capacity = - f64::from(config.user_visibility_cache_capacity) * config.conduit_cache_capacity_modifier; + f64::from(config.user_visibility_cache_capacity) * config.cache_capacity_modifier; Ok(Arc::new(Self { db: Data::new(args.db), diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index 4b4ea7d4..422c562b 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -55,7 +55,7 @@ pub struct Service { impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let config = &args.server.config; - let cache_capacity = f64::from(config.stateinfo_cache_capacity) * config.conduit_cache_capacity_modifier; + let cache_capacity = f64::from(config.stateinfo_cache_capacity) * config.cache_capacity_modifier; Ok(Arc::new(Self { db: Data::new(args.db), stateinfo_cache: StdMutex::new(LruCache::new(usize_from_f64(cache_capacity)?)), From bdd58454907690ca0f9b827b21fd062e1a5feb64 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 13 Jul 2024 22:15:16 -0400 Subject: [PATCH 134/158] docs: suggest recursive chmod Signed-off-by: strawberry --- docs/deploying/generic.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/deploying/generic.md b/docs/deploying/generic.md index 75b285d1..0cbd8ef8 100644 --- a/docs/deploying/generic.md +++ b/docs/deploying/generic.md @@ -23,7 +23,7 @@ Otherwise, follow standard Rust project build guides (installing git and cloning While conduwuit can run as any user it is better to use dedicated users for different services. This also allows you to make sure that the file permissions are correctly set up. -In Debian or RHEL, you can use this command to create a conduwuit user: +In Debian or Fedora/RHEL, you can use this command to create a conduwuit user: ```bash sudo adduser --system conduwuit --group --disabled-login --no-create-home @@ -53,13 +53,11 @@ RocksDB is the only supported database backend. ## Setting the correct file permissions -If you are using a dedicated user for conduwuit, you will need to allow it to read the config. To do that you can run this command on - -Debian or RHEL: +If you are using a dedicated user for conduwuit, you will need to allow it to read the config. To do that you can run this: ```bash sudo chown -R root:root /etc/conduwuit -sudo chmod 755 /etc/conduwuit +sudo chmod -R 755 /etc/conduwuit ``` If you use the default database path you also need to run this: From 9370e93a8dd2e9cc72c6bc48b7689146bd3a4405 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 13 Jul 2024 23:03:29 -0400 Subject: [PATCH 135/158] nix: try to make x86-64-v2 optimised builds if target is x86_64 Signed-off-by: strawberry --- nix/pkgs/main/cross-compilation-env.nix | 3 +++ nix/pkgs/main/default.nix | 19 +++++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/nix/pkgs/main/cross-compilation-env.nix b/nix/pkgs/main/cross-compilation-env.nix index 83fe6ed6..70d0e51c 100644 --- a/nix/pkgs/main/cross-compilation-env.nix +++ b/nix/pkgs/main/cross-compilation-env.nix @@ -13,6 +13,9 @@ lib.optionalAttrs stdenv.hostPlatform.isStatic { lib.concatStringsSep " " ([] + ++ lib.optionals + stdenv.targetPlatform.isx86_64 + [ "-C" "target-cpu=x86-64-v2" ] # This disables PIE for static builds, which isn't great in terms # of security. Unfortunately, my hand is forced because nixpkgs' # `libstdc++.a` is built without `-fPIE`, which precludes us from diff --git a/nix/pkgs/main/default.nix b/nix/pkgs/main/default.nix index ffb1de96..6e3e82d7 100644 --- a/nix/pkgs/main/default.nix +++ b/nix/pkgs/main/default.nix @@ -71,10 +71,23 @@ buildDepsOnlyEnv = # which breaks Darwin entirely enableLiburing = enableLiburing; }).overrideAttrs (old: { - # TODO: static rocksdb fails to build on darwin + # TODO: static rocksdb fails to build on darwin, also see # build log at meta.broken = stdenv.hostPlatform.isStatic && stdenv.isDarwin; + enableLiburing = enableLiburing; + + sse42Support = stdenv.targetPlatform.isx86_64; + + cmakeFlags = if stdenv.targetPlatform.isx86_64 + then lib.subtractLists [ "-DPORTABLE=1" ] old.cmakeFlags + ++ lib.optionals stdenv.targetPlatform.isx86_64 [ + "-DPORTABLE=x86-64-v2" + "-DUSE_SSE=1" + "-DHAVE_SSE=1" + "-DHAVE_SSE42=1" + ] + else old.cmakeFlags; }); in { @@ -101,7 +114,9 @@ buildPackageEnv = { # Only needed in static stdenv because these are transitive dependencies of rocksdb CARGO_BUILD_RUSTFLAGS = buildDepsOnlyEnv.CARGO_BUILD_RUSTFLAGS + lib.optionalString (enableLiburing && stdenv.hostPlatform.isStatic) - " -L${lib.getLib liburing}/lib -luring"; + " -L${lib.getLib liburing}/lib -luring" + + lib.optionalString stdenv.targetPlatform.isx86_64 + " -Ctarget-cpu=x86-64-v2"; }; From df28359a197ec09cbefbe3be496a8eb19597f553 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 13 Jul 2024 23:23:40 -0400 Subject: [PATCH 136/158] nix: dont build rocksdb core tools (ldb / sst_dump) (`-DWITH_CORE_TOOLS=1`) Signed-off-by: strawberry --- flake.nix | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index 0024e176..e34aaad0 100644 --- a/flake.nix +++ b/flake.nix @@ -44,10 +44,21 @@ # we have this already at https://github.com/girlbossceo/rocksdb/commit/a935c0273e1ba44eacf88ce3685a9b9831486155 # unsetting this so i don't have to revert it and make this nix exclusive patches = []; - # no real reason to have snappy, no one uses this cmakeFlags = pkgs.lib.subtractLists - [ "-DWITH_SNAPPY=1" ] - old.cmakeFlags; + [ + # no real reason to have snappy, no one uses this + "-DWITH_SNAPPY=1" + # we dont need to use ldb or sst_dump (core_tools) + "-DWITH_CORE_TOOLS=1" + ] + old.cmakeFlags + ++ [ "-DWITH_CORE_TOOLS=0" ]; + + # outputs has "tools" which we dont need or use + outputs = [ "out" ]; + + # preInstall hooks has stuff for messing with ldb/sst_dump which we dont need or use + preInstall = ""; }); # TODO: remove once https://github.com/NixOS/nixpkgs/pull/314945 is available liburing = pkgs.liburing.overrideAttrs (old: { From 649e9da1f8f6404123f240af99cd801a7d16ed3a Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 13 Jul 2024 23:24:25 -0400 Subject: [PATCH 137/158] nix: try to make armv8.2-a / cortex-a55 optimised builds if target is aarch64 Signed-off-by: strawberry --- nix/pkgs/main/cross-compilation-env.nix | 3 +++ nix/pkgs/main/default.nix | 10 +++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/nix/pkgs/main/cross-compilation-env.nix b/nix/pkgs/main/cross-compilation-env.nix index 70d0e51c..75b5862f 100644 --- a/nix/pkgs/main/cross-compilation-env.nix +++ b/nix/pkgs/main/cross-compilation-env.nix @@ -16,6 +16,9 @@ lib.optionalAttrs stdenv.hostPlatform.isStatic { ++ lib.optionals stdenv.targetPlatform.isx86_64 [ "-C" "target-cpu=x86-64-v2" ] + ++ lib.optionals + stdenv.targetPlatform.isAarch64 + [ "-C" "target-cpu=cortex-a55" ] # cortex-a55 == ARMv8.2-a # This disables PIE for static builds, which isn't great in terms # of security. Unfortunately, my hand is forced because nixpkgs' # `libstdc++.a` is built without `-fPIE`, which precludes us from diff --git a/nix/pkgs/main/default.nix b/nix/pkgs/main/default.nix index 6e3e82d7..629957c8 100644 --- a/nix/pkgs/main/default.nix +++ b/nix/pkgs/main/default.nix @@ -87,6 +87,12 @@ buildDepsOnlyEnv = "-DHAVE_SSE=1" "-DHAVE_SSE42=1" ] + else if stdenv.targetPlatform.isAarch64 + then lib.subtractLists [ "-DPORTABLE=1" ] old.cmakeFlags + ++ lib.optionals stdenv.targetPlatform.isAarch64 [ + # cortex-a55 == ARMv8.2-a + "-DPORTABLE=armv8.2-a" + ] else old.cmakeFlags; }); in @@ -116,7 +122,9 @@ buildPackageEnv = { + lib.optionalString (enableLiburing && stdenv.hostPlatform.isStatic) " -L${lib.getLib liburing}/lib -luring" + lib.optionalString stdenv.targetPlatform.isx86_64 - " -Ctarget-cpu=x86-64-v2"; + " -Ctarget-cpu=x86-64-v2" + + lib.optionalString stdenv.targetPlatform.isAarch64 + " -Ctarget-cpu=cortex-a55"; # cortex-a55 == ARMv8.2-a }; From f0c63c539bf1fd7347e2e5bddec7e6e9da7cde1d Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 14 Jul 2024 00:11:37 -0400 Subject: [PATCH 138/158] nix: dont build more unnecessary tests/tools for rocksdb Signed-off-by: strawberry --- flake.nix | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index e34aaad0..8ac5836f 100644 --- a/flake.nix +++ b/flake.nix @@ -50,9 +50,18 @@ "-DWITH_SNAPPY=1" # we dont need to use ldb or sst_dump (core_tools) "-DWITH_CORE_TOOLS=1" + # we dont need to build rocksdb tests + "-DWITH_TESTS=1" ] old.cmakeFlags - ++ [ "-DWITH_CORE_TOOLS=0" ]; + ++ [ + # we dont need to use ldb or sst_dump (core_tools) + "-DWITH_CORE_TOOLS=0" + # we dont need trace tools + "-DWITH_TRACE_TOOLS=0" + # we dont need to build rocksdb tests + "-DWITH_TESTS=0" + ]; # outputs has "tools" which we dont need or use outputs = [ "out" ]; From 188fa5a07397a01137bd1402ed7ae7b826ee4b27 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 14 Jul 2024 00:14:23 -0400 Subject: [PATCH 139/158] patch rustyline-async and use no default features Signed-off-by: strawberry --- Cargo.toml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f9696252..104535c8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -405,8 +405,8 @@ features = [ ] [workspace.dependencies.rustyline-async] -git = "https://github.com/girlbossceo/rustyline-async" -rev = "de26100b0db03e419a3d8e1dd26895d170d1fe50" +version = "0.4.2" +default-features = false [workspace.dependencies.termimad] version = "0.29.4" @@ -442,6 +442,12 @@ rev = "b348dca742af641c47bc390261f60711c2af573c" git = "https://github.com/girlbossceo/axum-server" rev = "8e3368d899079818934e61cc9c839abcbbcada8a" +# adds a tab completion callback: https://github.com/girlbossceo/rustyline-async/commit/de26100b0db03e419a3d8e1dd26895d170d1fe50 +# adds event for CTRL+\: https://github.com/girlbossceo/rustyline-async/commit/67d8c49aeac03a5ef4e818f663eaa94dd7bf339b +[patch.crates-io.rustyline-async] +git = "https://github.com/girlbossceo/rustyline-async" +rev = "de26100b0db03e419a3d8e1dd26895d170d1fe50" + # # Our crates # From 1797fec3c9109a8b02c782934a8677ce3b6f21ea Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 14 Jul 2024 00:38:29 -0400 Subject: [PATCH 140/158] nix: disable USE_RTTI for rocksdb Signed-off-by: strawberry --- flake.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/flake.nix b/flake.nix index 8ac5836f..a56d1bc0 100644 --- a/flake.nix +++ b/flake.nix @@ -52,6 +52,8 @@ "-DWITH_CORE_TOOLS=1" # we dont need to build rocksdb tests "-DWITH_TESTS=1" + # we use rust-rocksdb via C interface and dont need C++ RTTI + "-DUSE_RTTI=1" ] old.cmakeFlags ++ [ @@ -61,6 +63,8 @@ "-DWITH_TRACE_TOOLS=0" # we dont need to build rocksdb tests "-DWITH_TESTS=0" + # we use rust-rocksdb via C interface and dont need C++ RTTI + "-DUSE_RTTI=0" ]; # outputs has "tools" which we dont need or use From bdf2de076a0af8c5ac795bbabaa45feb6cc206bf Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 14 Jul 2024 00:47:56 -0400 Subject: [PATCH 141/158] ci: set NIX_CONFIG with our binary caches for the earlier CI steps Signed-off-by: strawberry --- .github/workflows/ci.yml | 7 +++++-- .github/workflows/documentation.yml | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 66ecdce8..19aa68cd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,8 +38,11 @@ env: # Custom nix binary cache if fork is being used ATTIC_ENDPOINT: ${{ vars.ATTIC_ENDPOINT }} ATTIC_PUBLIC_KEY: ${{ vars.ATTIC_PUBLIC_KEY }} - # Get error output from nix that we can actually use - NIX_CONFIG: show-trace = true + # Get error output from nix that we can actually use, and use our binary caches for the earlier CI steps + NIX_CONFIG: | + show-trace = true + extra-substituters = https://attic.kennel.juneis.dog/conduit https://attic.kennel.juneis.dog/conduwuit https://cache.lix.systems https://conduwuit.cachix.org + extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg= permissions: packages: write diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index ed0e898d..cab1710f 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -21,8 +21,11 @@ env: # Custom nix binary cache if fork is being used ATTIC_ENDPOINT: ${{ vars.ATTIC_ENDPOINT }} ATTIC_PUBLIC_KEY: ${{ vars.ATTIC_PUBLIC_KEY }} - # Get error output from nix that we can actually use - NIX_CONFIG: show-trace = true + # Get error output from nix that we can actually use, and use our binary caches for the earlier CI steps + NIX_CONFIG: | + show-trace = true + extra-substituters = https://attic.kennel.juneis.dog/conduit https://attic.kennel.juneis.dog/conduwuit https://cache.lix.systems https://conduwuit.cachix.org + extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg= # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. From e2280aa1a53c9403fffe53e866c641f9dbbafb66 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 14 Jul 2024 01:33:38 -0400 Subject: [PATCH 142/158] ci: try removing --impure Signed-off-by: strawberry --- .github/workflows/ci.yml | 4 ++-- .github/workflows/documentation.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 19aa68cd..9def2cf9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,7 +118,7 @@ jobs: - name: Prepare build environment run: | echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc" - nix profile install --impure --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv + nix profile install --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv direnv allow nix develop .#all-features --command true @@ -240,7 +240,7 @@ jobs: - name: Prepare build environment run: | echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc" - nix profile install --impure --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv + nix profile install --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv direnv allow nix develop .#all-features --command true diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index cab1710f..c6f9133a 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -101,7 +101,7 @@ jobs: - name: Prepare build environment run: | echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc" - nix profile install --impure --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv + nix profile install --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv direnv allow nix develop --command true From bda44b16b11d27dc83d45d8a5af25a5e4e785327 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 14 Jul 2024 02:07:07 -0400 Subject: [PATCH 143/158] ci: run nix dynamic build test with `DIRENV_DEVSHELL=dynamic` Signed-off-by: strawberry --- engage.toml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/engage.toml b/engage.toml index d28ecff1..8bba0eaf 100644 --- a/engage.toml +++ b/engage.toml @@ -184,6 +184,10 @@ cargo test \ name = "nix-default" group = "tests" script = """ -bin/nix-build-and-cache just .#default -nix run -L .#default -- --help +env DIRENV_DEVSHELL=dynamic \ + direnv exec . \ + bin/nix-build-and-cache just .#default +env DIRENV_DEVSHELL=dynamic \ + direnv exec . \ + nix run -L .#default -- --help """ From 80ec0e31b19dcf19549b8fd0f1b70d562e99245c Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 14 Jul 2024 20:33:22 -0400 Subject: [PATCH 144/158] bump tracing fork Signed-off-by: strawberry --- Cargo.lock | 10 +++++----- Cargo.toml | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 12b96665..2c637a34 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4250,7 +4250,7 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" version = "0.1.40" -source = "git+https://github.com/girlbossceo/tracing?rev=b348dca742af641c47bc390261f60711c2af573c#b348dca742af641c47bc390261f60711c2af573c" +source = "git+https://github.com/girlbossceo/tracing?rev=4d78a14a5e03f539b8c6b475aefa08bb14e4de91#4d78a14a5e03f539b8c6b475aefa08bb14e4de91" dependencies = [ "log", "pin-project-lite", @@ -4261,7 +4261,7 @@ dependencies = [ [[package]] name = "tracing-attributes" version = "0.1.27" -source = "git+https://github.com/girlbossceo/tracing?rev=b348dca742af641c47bc390261f60711c2af573c#b348dca742af641c47bc390261f60711c2af573c" +source = "git+https://github.com/girlbossceo/tracing?rev=4d78a14a5e03f539b8c6b475aefa08bb14e4de91#4d78a14a5e03f539b8c6b475aefa08bb14e4de91" dependencies = [ "proc-macro2", "quote", @@ -4271,7 +4271,7 @@ dependencies = [ [[package]] name = "tracing-core" version = "0.1.32" -source = "git+https://github.com/girlbossceo/tracing?rev=b348dca742af641c47bc390261f60711c2af573c#b348dca742af641c47bc390261f60711c2af573c" +source = "git+https://github.com/girlbossceo/tracing?rev=4d78a14a5e03f539b8c6b475aefa08bb14e4de91#4d78a14a5e03f539b8c6b475aefa08bb14e4de91" dependencies = [ "once_cell", "valuable", @@ -4291,7 +4291,7 @@ dependencies = [ [[package]] name = "tracing-log" version = "0.2.0" -source = "git+https://github.com/girlbossceo/tracing?rev=b348dca742af641c47bc390261f60711c2af573c#b348dca742af641c47bc390261f60711c2af573c" +source = "git+https://github.com/girlbossceo/tracing?rev=4d78a14a5e03f539b8c6b475aefa08bb14e4de91#4d78a14a5e03f539b8c6b475aefa08bb14e4de91" dependencies = [ "log", "once_cell", @@ -4319,7 +4319,7 @@ dependencies = [ [[package]] name = "tracing-subscriber" version = "0.3.18" -source = "git+https://github.com/girlbossceo/tracing?rev=b348dca742af641c47bc390261f60711c2af573c#b348dca742af641c47bc390261f60711c2af573c" +source = "git+https://github.com/girlbossceo/tracing?rev=4d78a14a5e03f539b8c6b475aefa08bb14e4de91#4d78a14a5e03f539b8c6b475aefa08bb14e4de91" dependencies = [ "matchers", "nu-ansi-term", diff --git a/Cargo.toml b/Cargo.toml index 104535c8..db48b42e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -425,16 +425,16 @@ version = "0.1" # https://github.com/girlbossceo/tracing/commit/b348dca742af641c47bc390261f60711c2af573c [patch.crates-io.tracing-subscriber] git = "https://github.com/girlbossceo/tracing" -rev = "b348dca742af641c47bc390261f60711c2af573c" +rev = "4d78a14a5e03f539b8c6b475aefa08bb14e4de91" [patch.crates-io.tracing] git = "https://github.com/girlbossceo/tracing" -rev = "b348dca742af641c47bc390261f60711c2af573c" +rev = "4d78a14a5e03f539b8c6b475aefa08bb14e4de91" [patch.crates-io.tracing-core] git = "https://github.com/girlbossceo/tracing" -rev = "b348dca742af641c47bc390261f60711c2af573c" +rev = "4d78a14a5e03f539b8c6b475aefa08bb14e4de91" [patch.crates-io.tracing-log] git = "https://github.com/girlbossceo/tracing" -rev = "b348dca742af641c47bc390261f60711c2af573c" +rev = "4d78a14a5e03f539b8c6b475aefa08bb14e4de91" # fixes hyper graceful shutdowns [https://github.com/programatik29/axum-server/issues/114] # https://github.com/girlbossceo/axum-server/commit/8e3368d899079818934e61cc9c839abcbbcada8a From f79bd2ac72e4135d170b526657027b458a921da2 Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 15 Jul 2024 21:45:10 -0400 Subject: [PATCH 145/158] bump ruma-identifiers-validation to c51ccb2c68d2e3557eb12b1a49036531711ec0e5 Signed-off-by: strawberry --- Cargo.lock | 17 ++++------------- Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2c637a34..61cf48bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -769,7 +769,7 @@ dependencies = [ "regex", "reqwest", "ruma", - "ruma-identifiers-validation 0.9.5 (git+https://github.com/girlbossceo/ruwuma?rev=fd686e77950680462377c9105dfb4136dd49c7a0)", + "ruma-identifiers-validation", "rustyline-async", "serde", "serde_json", @@ -2988,7 +2988,7 @@ dependencies = [ "percent-encoding", "rand", "regex", - "ruma-identifiers-validation 0.9.5 (git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5)", + "ruma-identifiers-validation", "ruma-macros", "serde", "serde_html_form", @@ -3015,7 +3015,7 @@ dependencies = [ "pulldown-cmark", "regex", "ruma-common", - "ruma-identifiers-validation 0.9.5 (git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5)", + "ruma-identifiers-validation", "ruma-macros", "serde", "serde_json", @@ -3046,15 +3046,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "ruma-identifiers-validation" -version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=fd686e77950680462377c9105dfb4136dd49c7a0#fd686e77950680462377c9105dfb4136dd49c7a0" -dependencies = [ - "js_int", - "thiserror", -] - [[package]] name = "ruma-identity-service-api" version = "0.9.0" @@ -3074,7 +3065,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "ruma-identifiers-validation 0.9.5 (git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5)", + "ruma-identifiers-validation", "serde", "syn 2.0.71", "toml", diff --git a/Cargo.toml b/Cargo.toml index db48b42e..7c17d0f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -309,7 +309,7 @@ features = [ [workspace.dependencies.ruma-identifiers-validation] git = "https://github.com/girlbossceo/ruwuma" -rev = "fd686e77950680462377c9105dfb4136dd49c7a0" +rev = "c51ccb2c68d2e3557eb12b1a49036531711ec0e5" [workspace.dependencies.rust-rocksdb] path = "deps/rust-rocksdb" From 59503553485da33d4521612917f103e9ad88e6f2 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 15 Jul 2024 02:50:09 +0000 Subject: [PATCH 146/158] break from wait loop on empty taskset Signed-off-by: Jason Volk --- src/router/serve/unix.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/router/serve/unix.rs b/src/router/serve/unix.rs index b5938673..fb011f18 100644 --- a/src/router/serve/unix.rs +++ b/src/router/serve/unix.rs @@ -127,8 +127,8 @@ async fn fini(server: &Arc, listener: UnixListener, mut tasks: JoinSet<( debug!("Waiting for requests to finish..."); while server.metrics.requests_spawn_active.load(Ordering::Relaxed) > 0 { tokio::select! { - _ = tasks.join_next() => {} - () = sleep(FINI_POLL_INTERVAL) => {} + task = tasks.join_next() => if task.is_none() { break; }, + () = sleep(FINI_POLL_INTERVAL) => {}, } } From c42cb90dd3ccd8f3b531c75c46b08c7891186cde Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 15 Jul 2024 01:49:30 +0000 Subject: [PATCH 147/158] simplify result handler / 405 error interposition Signed-off-by: Jason Volk --- src/core/error/err.rs | 12 ++++++--- src/core/error/mod.rs | 7 ++--- src/core/error/response.rs | 54 +++++++++++++++++++++++++++----------- src/router/request.rs | 31 ++++++---------------- src/router/router.rs | 4 +-- 5 files changed, 60 insertions(+), 48 deletions(-) diff --git a/src/core/error/err.rs b/src/core/error/err.rs index ea596644..66634f2b 100644 --- a/src/core/error/err.rs +++ b/src/core/error/err.rs @@ -49,14 +49,16 @@ macro_rules! err { $crate::$level!($($args),*); $crate::error::Error::Request( ::ruma::api::client::error::ErrorKind::forbidden(), - $crate::format_maybe!($($args),*) + $crate::format_maybe!($($args),*), + ::http::StatusCode::BAD_REQUEST ) }}; (Request(Forbidden($($args:expr),*))) => { $crate::error::Error::Request( ::ruma::api::client::error::ErrorKind::forbidden(), - $crate::format_maybe!($($args),*) + $crate::format_maybe!($($args),*), + ::http::StatusCode::BAD_REQUEST ) }; @@ -64,14 +66,16 @@ macro_rules! err { $crate::$level!($($args),*); $crate::error::Error::Request( ::ruma::api::client::error::ErrorKind::$variant, - $crate::format_maybe!($($args),*) + $crate::format_maybe!($($args),*), + ::http::StatusCode::BAD_REQUEST ) }}; (Request($variant:ident($($args:expr),*))) => { $crate::error::Error::Request( ::ruma::api::client::error::ErrorKind::$variant, - $crate::format_maybe!($($args),*) + $crate::format_maybe!($($args),*), + ::http::StatusCode::BAD_REQUEST ) }; diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index 069fe60e..1f0d56c1 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -68,7 +68,7 @@ pub enum Error { #[error("{0}: {1}")] BadRequest(ruma::api::client::error::ErrorKind, &'static str), //TODO: remove #[error("{0}: {1}")] - Request(ruma::api::client::error::ErrorKind, Cow<'static, str>), + Request(ruma::api::client::error::ErrorKind, Cow<'static, str>, http::StatusCode), #[error("from {0}: {1}")] Redaction(ruma::OwnedServerName, ruma::canonical_json::RedactionError), #[error("Remote server {0} responded with: {1}")] @@ -120,7 +120,7 @@ impl Error { match self { Self::Federation(_, error) => response::ruma_error_kind(error).clone(), - Self::BadRequest(kind, _) | Self::Request(kind, _) => kind.clone(), + Self::BadRequest(kind, ..) | Self::Request(kind, ..) => kind.clone(), _ => Unknown, } } @@ -128,7 +128,8 @@ impl Error { pub fn status_code(&self) -> http::StatusCode { match self { Self::Federation(_, ref error) | Self::RumaError(ref error) => error.status_code, - Self::BadRequest(ref kind, _) | Self::Request(ref kind, _) => response::bad_request_code(kind), + Self::Request(ref kind, _, code) => response::status_code(kind, *code), + Self::BadRequest(ref kind, ..) => response::bad_request_code(kind), Self::Conflict(_) => http::StatusCode::CONFLICT, _ => http::StatusCode::INTERNAL_SERVER_ERROR, } diff --git a/src/core/error/response.rs b/src/core/error/response.rs index 4ea76e26..7568a1c0 100644 --- a/src/core/error/response.rs +++ b/src/core/error/response.rs @@ -1,7 +1,13 @@ use bytes::BytesMut; use http::StatusCode; use http_body_util::Full; -use ruma::api::{client::uiaa::UiaaResponse, OutgoingResponse}; +use ruma::api::{ + client::{ + error::{ErrorBody, ErrorKind}, + uiaa::UiaaResponse, + }, + OutgoingResponse, +}; use super::Error; use crate::error; @@ -25,7 +31,7 @@ impl From for UiaaResponse { return Self::AuthResponse(uiaainfo); } - let body = ruma::api::client::error::ErrorBody::Standard { + let body = ErrorBody::Standard { kind: error.kind(), message: error.message(), }; @@ -37,10 +43,33 @@ impl From for UiaaResponse { } } -pub(super) fn bad_request_code(kind: &ruma::api::client::error::ErrorKind) -> StatusCode { - use ruma::api::client::error::ErrorKind::*; +pub(super) fn status_code(kind: &ErrorKind, hint: StatusCode) -> StatusCode { + if hint == StatusCode::BAD_REQUEST { + bad_request_code(kind) + } else { + hint + } +} + +pub(super) fn bad_request_code(kind: &ErrorKind) -> StatusCode { + use ErrorKind::*; match kind { + // 429 + LimitExceeded { + .. + } => StatusCode::TOO_MANY_REQUESTS, + + // 413 + TooLarge => StatusCode::PAYLOAD_TOO_LARGE, + + // 405 + Unrecognized => StatusCode::METHOD_NOT_ALLOWED, + + // 404 + NotFound => StatusCode::NOT_FOUND, + + // 403 GuestAccessForbidden | ThreepidAuthFailed | UserDeactivated @@ -52,26 +81,20 @@ pub(super) fn bad_request_code(kind: &ruma::api::client::error::ErrorKind) -> St .. } => StatusCode::FORBIDDEN, + // 401 UnknownToken { .. } | MissingToken | Unauthorized => StatusCode::UNAUTHORIZED, - LimitExceeded { - .. - } => StatusCode::TOO_MANY_REQUESTS, - - TooLarge => StatusCode::PAYLOAD_TOO_LARGE, - - NotFound | Unrecognized => StatusCode::NOT_FOUND, - + // 400 _ => StatusCode::BAD_REQUEST, } } pub(super) fn ruma_error_message(error: &ruma::api::client::error::Error) -> String { - if let ruma::api::client::error::ErrorBody::Standard { + if let ErrorBody::Standard { message, .. } = &error.body @@ -82,7 +105,6 @@ pub(super) fn ruma_error_message(error: &ruma::api::client::error::Error) -> Str format!("{error}") } -pub(super) fn ruma_error_kind(e: &ruma::api::client::error::Error) -> &ruma::api::client::error::ErrorKind { - e.error_kind() - .unwrap_or(&ruma::api::client::error::ErrorKind::Unknown) +pub(super) fn ruma_error_kind(e: &ruma::api::client::error::Error) -> &ErrorKind { + e.error_kind().unwrap_or(&ErrorKind::Unknown) } diff --git a/src/router/request.rs b/src/router/request.rs index ae739984..4d5c61e7 100644 --- a/src/router/request.rs +++ b/src/router/request.rs @@ -4,9 +4,8 @@ use axum::{ extract::State, response::{IntoResponse, Response}, }; -use conduit::{debug, debug_error, debug_warn, defer, error, trace, Error, Result, Server}; +use conduit::{debug, debug_error, debug_warn, defer, err, error, trace, Result, Server}; use http::{Method, StatusCode, Uri}; -use ruma::api::client::error::{Error as RumaError, ErrorBody, ErrorKind}; #[tracing::instrument(skip_all, level = "debug")] pub(crate) async fn spawn( @@ -58,33 +57,13 @@ pub(crate) async fn handle( trace!(active, finished, "leave"); }}; - let method = req.method().clone(); let uri = req.uri().clone(); + let method = req.method().clone(); let result = next.run(req).await; handle_result(&method, &uri, result) } fn handle_result(method: &Method, uri: &Uri, result: Response) -> Result { - handle_result_log(method, uri, &result); - match result.status() { - StatusCode::METHOD_NOT_ALLOWED => handle_result_405(method, uri, &result), - _ => Ok(result), - } -} - -fn handle_result_405(_method: &Method, _uri: &Uri, result: &Response) -> Result { - let error = Error::RumaError(RumaError { - status_code: result.status(), - body: ErrorBody::Standard { - kind: ErrorKind::Unrecognized, - message: "M_UNRECOGNIZED: Method not allowed for endpoint".to_owned(), - }, - }); - - Ok(error.into_response()) -} - -fn handle_result_log(method: &Method, uri: &Uri, result: &Response) { let status = result.status(); let reason = status.canonical_reason().unwrap_or("Unknown Reason"); let code = status.as_u16(); @@ -97,4 +76,10 @@ fn handle_result_log(method: &Method, uri: &Uri, result: &Response) { } else { trace!(method = ?method, uri = ?uri, "{code} {reason}"); } + + if status == StatusCode::METHOD_NOT_ALLOWED { + return Ok(err!(Request(Unrecognized("Method Not Allowed"))).into_response()); + } + + Ok(result) } diff --git a/src/router/router.rs b/src/router/router.rs index da31ffea..2f6d5c46 100644 --- a/src/router/router.rs +++ b/src/router/router.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use axum::{response::IntoResponse, routing::get, Router}; use conduit::{Error, Server}; use conduit_service as service; -use http::Uri; +use http::{StatusCode, Uri}; use ruma::api::client::error::ErrorKind; extern crate conduit_api as api; @@ -19,7 +19,7 @@ pub(crate) fn build(server: &Arc) -> Router { } async fn not_found(_uri: Uri) -> impl IntoResponse { - Error::BadRequest(ErrorKind::Unrecognized, "Unrecognized request") + Error::Request(ErrorKind::Unrecognized, "Not Found".into(), StatusCode::NOT_FOUND) } async fn it_works() -> &'static str { "hewwo from conduwuit woof!" } From 720fbd09c212d1fdbf93484c27de9abcd239127a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 14 Jul 2024 23:46:03 +0000 Subject: [PATCH 148/158] move routes into api router top level Signed-off-by: Jason Volk --- src/api/mod.rs | 9 ++++----- src/api/{routes.rs => router.rs} | 32 +++++++++++++++++++----------- src/api/router/{mod.rs => args.rs} | 18 ++++++----------- src/router/router.rs | 2 +- 4 files changed, 31 insertions(+), 30 deletions(-) rename src/api/{routes.rs => router.rs} (93%) rename src/api/router/{mod.rs => args.rs} (90%) diff --git a/src/api/mod.rs b/src/api/mod.rs index 6adf2d39..dda37caf 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -1,15 +1,14 @@ pub mod client; -mod router; -pub mod routes; +pub mod router; pub mod server; extern crate conduit_core as conduit; extern crate conduit_service as service; -pub(crate) use conduit::{debug_info, debug_warn, utils, Error, Result}; -pub(crate) use service::{pdu::PduEvent, services, user_is_local}; +pub(crate) use conduit::{debug_info, debug_warn, pdu::PduEvent, utils, Error, Result}; +pub(crate) use service::{services, user_is_local}; -pub(crate) use self::router::{Ruma, RumaResponse}; +pub(crate) use crate::router::{Ruma, RumaResponse}; conduit::mod_ctor! {} conduit::mod_dtor! {} diff --git a/src/api/routes.rs b/src/api/router.rs similarity index 93% rename from src/api/routes.rs rename to src/api/router.rs index 3a8b2c74..9b6f62a5 100644 --- a/src/api/routes.rs +++ b/src/api/router.rs @@ -1,3 +1,9 @@ +mod args; +mod auth; +mod handler; +mod request; +mod response; + use axum::{ response::IntoResponse, routing::{any, get, post}, @@ -7,7 +13,9 @@ use conduit::{err, Error, Server}; use http::Uri; use ruma::api::client::error::ErrorKind; -use crate::{client, router::RouterExt, server}; +use self::handler::RouterExt; +pub(super) use self::{ar::Ruma, response::RumaResponse}; +use crate::{client, server}; pub fn build(router: Router, server: &Server) -> Router { let config = &server.config; @@ -95,7 +103,7 @@ pub fn build(router: Router, server: &Server) -> Router { .ruma_route(client::get_member_events_route) .ruma_route(client::get_protocols_route) .route("/_matrix/client/unstable/thirdparty/protocols", - get(client::get_protocols_route_unstable)) + get(client::get_protocols_route_unstable)) .ruma_route(client::send_message_event_route) .ruma_route(client::send_state_event_for_key_route) .ruma_route(client::get_state_events_route) @@ -180,15 +188,15 @@ pub fn build(router: Router, server: &Server) -> Router { .ruma_route(client::get_relating_events_with_rel_type_route) .ruma_route(client::get_relating_events_route) .ruma_route(client::get_hierarchy_route) - .ruma_route(client::get_mutual_rooms_route) - .ruma_route(client::get_room_summary) - .route( - "/_matrix/client/unstable/im.nheko.summary/rooms/:room_id_or_alias/summary", - get(client::get_room_summary_legacy) - ) - .ruma_route(client::well_known_support) - .ruma_route(client::well_known_client) - .route("/_conduwuit/server_version", get(client::conduwuit_server_version)) + .ruma_route(client::get_mutual_rooms_route) + .ruma_route(client::get_room_summary) + .route( + "/_matrix/client/unstable/im.nheko.summary/rooms/:room_id_or_alias/summary", + get(client::get_room_summary_legacy) + ) + .ruma_route(client::well_known_support) + .ruma_route(client::well_known_client) + .route("/_conduwuit/server_version", get(client::conduwuit_server_version)) .route("/_matrix/client/r0/rooms/:room_id/initialSync", get(initial_sync)) .route("/_matrix/client/v3/rooms/:room_id/initialSync", get(initial_sync)) .route("/client/server.json", get(client::syncv3_client_server_json)); @@ -233,7 +241,7 @@ pub fn build(router: Router, server: &Server) -> Router { } async fn initial_sync(_uri: Uri) -> impl IntoResponse { - Error::BadRequest(ErrorKind::GuestAccessForbidden, "Guest access not implemented") + err!(Request(GuestAccessForbidden("Guest access not implemented"))) } async fn federation_disabled() -> impl IntoResponse { err!(Config("allow_federation", "Federation is disabled.")) } diff --git a/src/api/router/mod.rs b/src/api/router/args.rs similarity index 90% rename from src/api/router/mod.rs rename to src/api/router/args.rs index c3e08c5b..b54a6f80 100644 --- a/src/api/router/mod.rs +++ b/src/api/router/args.rs @@ -1,24 +1,18 @@ -mod auth; -mod handler; -mod request; -mod response; - use std::{mem, ops::Deref}; use axum::{async_trait, body::Body, extract::FromRequest}; use bytes::{BufMut, BytesMut}; -use conduit::{debug, debug_warn, trace, warn}; +use conduit::{debug, debug_warn, trace, warn, Error, Result}; use ruma::{ api::{client::error::ErrorKind, IncomingRequest}, CanonicalJsonValue, OwnedDeviceId, OwnedServerName, OwnedUserId, UserId, }; -use self::{auth::Auth, request::Request}; -pub(super) use self::{handler::RouterExt, response::RumaResponse}; -use crate::{service::appservice::RegistrationInfo, services, Error, Result}; +use super::{auth, auth::Auth, request, request::Request}; +use crate::{service::appservice::RegistrationInfo, services}; /// Extractor for Ruma request structs -pub(crate) struct Ruma { +pub(crate) struct Args { /// Request struct body pub(crate) body: T, @@ -44,7 +38,7 @@ pub(crate) struct Ruma { } #[async_trait] -impl FromRequest for Ruma +impl FromRequest for Args where T: IncomingRequest, { @@ -65,7 +59,7 @@ where } } -impl Deref for Ruma { +impl Deref for Args { type Target = T; fn deref(&self) -> &Self::Target { &self.body } diff --git a/src/router/router.rs b/src/router/router.rs index 2f6d5c46..7dfb089b 100644 --- a/src/router/router.rs +++ b/src/router/router.rs @@ -10,7 +10,7 @@ extern crate conduit_api as api; pub(crate) fn build(server: &Arc) -> Router { let state = service::services(); - let router = Router::new() + api::router::build(Router::new(), server) .route("/", get(it_works)) .fallback(not_found) .with_state(state); From 038b71fc9db286025614fc425add2e478abf79fc Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 15 Jul 2024 03:56:27 +0000 Subject: [PATCH 149/158] add state to router Signed-off-by: Jason Volk --- src/api/mod.rs | 1 + src/api/router.rs | 9 +++++---- src/api/router/handler.rs | 16 ++++++++-------- src/router/router.rs | 11 +++++------ 4 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/api/mod.rs b/src/api/mod.rs index dda37caf..79382934 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -8,6 +8,7 @@ extern crate conduit_service as service; pub(crate) use conduit::{debug_info, debug_warn, pdu::PduEvent, utils, Error, Result}; pub(crate) use service::{services, user_is_local}; +pub use crate::router::State; pub(crate) use crate::router::{Ruma, RumaResponse}; conduit::mod_ctor! {} diff --git a/src/api/router.rs b/src/api/router.rs index 9b6f62a5..761c173c 100644 --- a/src/api/router.rs +++ b/src/api/router.rs @@ -9,15 +9,16 @@ use axum::{ routing::{any, get, post}, Router, }; -use conduit::{err, Error, Server}; +use conduit::{err, Server}; use http::Uri; -use ruma::api::client::error::ErrorKind; use self::handler::RouterExt; -pub(super) use self::{ar::Ruma, response::RumaResponse}; +pub(super) use self::{args::Args as Ruma, response::RumaResponse}; use crate::{client, server}; -pub fn build(router: Router, server: &Server) -> Router { +pub type State = &'static service::Services; + +pub fn build(router: Router, server: &Server) -> Router { let config = &server.config; let router = router .ruma_route(client::get_supported_versions_route) diff --git a/src/api/router/handler.rs b/src/api/router/handler.rs index 73361989..d112ec58 100644 --- a/src/api/router/handler.rs +++ b/src/api/router/handler.rs @@ -10,7 +10,7 @@ use conduit::Result; use http::Method; use ruma::api::IncomingRequest; -use super::{Ruma, RumaResponse}; +use super::{Ruma, RumaResponse, State}; pub(in super::super) trait RouterExt { fn ruma_route(self, handler: H) -> Self @@ -18,7 +18,7 @@ pub(in super::super) trait RouterExt { H: RumaHandler; } -impl RouterExt for Router { +impl RouterExt for Router { fn ruma_route(self, handler: H) -> Self where H: RumaHandler, @@ -28,9 +28,9 @@ impl RouterExt for Router { } pub(in super::super) trait RumaHandler { - fn add_routes(&self, router: Router) -> Router; + fn add_routes(&self, router: Router) -> Router; - fn add_route(&self, router: Router, path: &str) -> Router; + fn add_route(&self, router: Router, path: &str) -> Router; } macro_rules! ruma_handler { @@ -41,17 +41,17 @@ macro_rules! ruma_handler { Req: IncomingRequest + Send + 'static, Ret: IntoResponse, Fut: Future> + Send, - Fun: FnOnce($($tx,)* Ruma) -> Fut + Clone + Send + Sync + 'static, - $( $tx: FromRequestParts<()> + Send + 'static, )* + Fun: FnOnce($($tx,)* Ruma,) -> Fut + Clone + Send + Sync + 'static, + $( $tx: FromRequestParts + Send + 'static, )* { - fn add_routes(&self, router: Router) -> Router { + fn add_routes(&self, router: Router) -> Router { Req::METADATA .history .all_paths() .fold(router, |router, path| self.add_route(router, path)) } - fn add_route(&self, router: Router, path: &str) -> Router { + fn add_route(&self, router: Router, path: &str) -> Router { let handle = self.clone(); let method = method_to_filter(&Req::METADATA.method); let action = |$($tx,)* req| async { handle($($tx,)* req).await.map(RumaResponse) }; diff --git a/src/router/router.rs b/src/router/router.rs index 7dfb089b..7c374b47 100644 --- a/src/router/router.rs +++ b/src/router/router.rs @@ -2,20 +2,19 @@ use std::sync::Arc; use axum::{response::IntoResponse, routing::get, Router}; use conduit::{Error, Server}; -use conduit_service as service; use http::{StatusCode, Uri}; use ruma::api::client::error::ErrorKind; extern crate conduit_api as api; +extern crate conduit_service as service; pub(crate) fn build(server: &Arc) -> Router { - let state = service::services(); - api::router::build(Router::new(), server) + let router = Router::::new(); + + api::router::build(router, server) .route("/", get(it_works)) .fallback(not_found) - .with_state(state); - - api::routes::build(router, server) + .with_state(service::services()) } async fn not_found(_uri: Uri) -> impl IntoResponse { From 838e4b9d8db24ee6533f3174cdcc56bfc247b293 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 15 Jul 2024 04:40:14 +0000 Subject: [PATCH 150/158] log more details for panic in tower handler Signed-off-by: Jason Volk --- src/router/layers.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/router/layers.rs b/src/router/layers.rs index 073940f1..67342eb3 100644 --- a/src/router/layers.rs +++ b/src/router/layers.rs @@ -5,7 +5,7 @@ use axum::{ Router, }; use axum_client_ip::SecureClientIpSource; -use conduit::{Result, Server}; +use conduit::{error, Result, Server}; use http::{ header::{self, HeaderName}, HeaderValue, Method, StatusCode, @@ -149,7 +149,7 @@ fn cors_layer(_server: &Server) -> CorsLayer { fn body_limit_layer(server: &Server) -> DefaultBodyLimit { DefaultBodyLimit::max(server.config.max_request_size) } #[allow(clippy::needless_pass_by_value)] -#[tracing::instrument(skip_all)] +#[tracing::instrument(skip_all, name = "panic")] fn catch_panic(err: Box) -> http::Response> { conduit_service::services() .server @@ -165,17 +165,17 @@ fn catch_panic(err: Box) -> http::Response Date: Mon, 24 Jun 2024 08:27:06 +0000 Subject: [PATCH 151/158] rename FileMeta::file to Option Signed-off-by: Jason Volk --- src/api/client/media.rs | 9 +++++--- src/service/media/mod.rs | 45 ++++++++++++++++++++++------------------ 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/src/api/client/media.rs b/src/api/client/media.rs index 9d3fbabe..39640b23 100644 --- a/src/api/client/media.rs +++ b/src/api/client/media.rs @@ -200,12 +200,13 @@ pub(crate) async fn get_content_route( let mxc = format!("mxc://{}/{}", body.server_name, body.media_id); if let Some(FileMeta { + content, content_type, - file, content_disposition, }) = services().media.get(&mxc).await? { let content_disposition = Some(make_content_disposition(&content_type, content_disposition, None)); + let file = content.expect("content"); Ok(get_content::v3::Response { file, @@ -282,8 +283,8 @@ pub(crate) async fn get_content_as_filename_route( let mxc = format!("mxc://{}/{}", body.server_name, body.media_id); if let Some(FileMeta { + content, content_type, - file, content_disposition, }) = services().media.get(&mxc).await? { @@ -293,6 +294,7 @@ pub(crate) async fn get_content_as_filename_route( Some(body.filename.clone()), )); + let file = content.expect("content"); Ok(get_content_as_filename::v3::Response { file, content_type, @@ -371,8 +373,8 @@ pub(crate) async fn get_content_thumbnail_route( let mxc = format!("mxc://{}/{}", body.server_name, body.media_id); if let Some(FileMeta { + content, content_type, - file, content_disposition, }) = services() .media @@ -388,6 +390,7 @@ pub(crate) async fn get_content_thumbnail_route( .await? { let content_disposition = Some(make_content_disposition(&content_type, content_disposition, None)); + let file = content.expect("content"); Ok(get_content_thumbnail::v3::Response { file, diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index 9b1cbe06..2f687925 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -20,10 +20,9 @@ use crate::services; #[derive(Debug)] pub struct FileMeta { - #[allow(dead_code)] - pub content_disposition: Option, + pub content: Option>, pub content_type: Option, - pub file: Vec, + pub content_disposition: Option, } #[derive(Serialize, Default)] @@ -132,16 +131,16 @@ impl Service { /// Downloads a file. pub async fn get(&self, mxc: &str) -> Result> { if let Ok((content_disposition, content_type, key)) = self.db.search_file_metadata(mxc, 0, 0) { - let mut file = Vec::new(); + let mut content = Vec::new(); let path = self.get_media_file(&key); BufReader::new(fs::File::open(path).await?) - .read_to_end(&mut file) + .read_to_end(&mut content) .await?; Ok(Some(FileMeta { - content_disposition, + content: Some(content), content_type, - file, + content_disposition, })) } else { Ok(None) @@ -283,29 +282,35 @@ impl Service { if let Ok((content_disposition, content_type, key)) = self.db.search_file_metadata(mxc, width, height) { // Using saved thumbnail - let mut file = Vec::new(); + let mut content = Vec::new(); let path = self.get_media_file(&key); - fs::File::open(path).await?.read_to_end(&mut file).await?; + fs::File::open(path) + .await? + .read_to_end(&mut content) + .await?; Ok(Some(FileMeta { - content_disposition, + content: Some(content), content_type, - file: file.clone(), + content_disposition, })) } else if let Ok((content_disposition, content_type, key)) = self.db.search_file_metadata(mxc, 0, 0) { // Generate a thumbnail - let mut file = Vec::new(); + let mut content = Vec::new(); let path = self.get_media_file(&key); - fs::File::open(path).await?.read_to_end(&mut file).await?; + fs::File::open(path) + .await? + .read_to_end(&mut content) + .await?; - if let Ok(image) = image::load_from_memory(&file) { + if let Ok(image) = image::load_from_memory(&content) { let original_width = image.width(); let original_height = image.height(); if width > original_width || height > original_height { return Ok(Some(FileMeta { - content_disposition, + content: Some(content), content_type, - file: file.clone(), + content_disposition, })); } @@ -350,16 +355,16 @@ impl Service { f.write_all(&thumbnail_bytes).await?; Ok(Some(FileMeta { - content_disposition, + content: Some(thumbnail_bytes), content_type, - file: thumbnail_bytes.clone(), + content_disposition, })) } else { // Couldn't parse file to generate thumbnail, send original Ok(Some(FileMeta { - content_disposition, + content: Some(content), content_type, - file: file.clone(), + content_disposition, })) } } else { From b903b46d16352e536e7fbecfbb2ab1f97ace8d4a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 15 Jul 2024 07:21:10 +0000 Subject: [PATCH 152/158] split thumbnailing related into unit Signed-off-by: Jason Volk --- src/service/media/data.rs | 18 +++- src/service/media/mod.rs | 159 ++---------------------------- src/service/media/thumbnail.rs | 171 +++++++++++++++++++++++++++++++++ 3 files changed, 195 insertions(+), 153 deletions(-) create mode 100644 src/service/media/thumbnail.rs diff --git a/src/service/media/data.rs b/src/service/media/data.rs index 186d502d..e5856bbf 100644 --- a/src/service/media/data.rs +++ b/src/service/media/data.rs @@ -12,6 +12,13 @@ pub(crate) struct Data { url_previews: Arc, } +#[derive(Debug)] +pub(super) struct Metadata { + pub(super) content_disposition: Option, + pub(super) content_type: Option, + pub(super) key: Vec, +} + impl Data { pub(super) fn new(db: &Arc) -> Self { Self { @@ -104,9 +111,7 @@ impl Data { Ok(keys) } - pub(super) fn search_file_metadata( - &self, mxc: &str, width: u32, height: u32, - ) -> Result<(Option, Option, Vec)> { + pub(super) fn search_file_metadata(&self, mxc: &str, width: u32, height: u32) -> Result { let mut prefix = mxc.as_bytes().to_vec(); prefix.push(0xFF); prefix.extend_from_slice(&width.to_be_bytes()); @@ -141,7 +146,12 @@ impl Data { .map_err(|_| Error::bad_database("Content Disposition in mediaid_file is invalid unicode."))?, ) }; - Ok((content_disposition, content_type, key)) + + Ok(Metadata { + content_disposition, + content_type, + key, + }) } /// Gets all the media keys in our database (this includes all the metadata diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index 2f687925..c969d1e8 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -1,13 +1,13 @@ mod data; mod tests; +mod thumbnail; -use std::{collections::HashMap, io::Cursor, num::Saturating as Sat, path::PathBuf, sync::Arc, time::SystemTime}; +use std::{collections::HashMap, path::PathBuf, sync::Arc, time::SystemTime}; use async_trait::async_trait; use base64::{engine::general_purpose, Engine as _}; -use conduit::{checked, debug, debug_error, error, utils, Error, Result, Server}; -use data::Data; -use image::imageops::FilterType; +use conduit::{debug, debug_error, err, error, utils, Result, Server}; +use data::{Data, Metadata}; use ruma::{OwnedMxcUri, OwnedUserId}; use serde::Serialize; use tokio::{ @@ -107,30 +107,14 @@ impl Service { } } - /// Uploads or replaces a file thumbnail. - #[allow(clippy::too_many_arguments)] - pub async fn upload_thumbnail( - &self, sender_user: Option, mxc: &str, content_disposition: Option<&str>, - content_type: Option<&str>, width: u32, height: u32, file: &[u8], - ) -> Result<()> { - let key = if let Some(user) = sender_user { - self.db - .create_file_metadata(Some(user.as_str()), mxc, width, height, content_disposition, content_type)? - } else { - self.db - .create_file_metadata(None, mxc, width, height, content_disposition, content_type)? - }; - - //TODO: Dangling metadata in database if creation fails - let mut f = self.create_media_file(&key).await?; - f.write_all(file).await?; - - Ok(()) - } - /// Downloads a file. pub async fn get(&self, mxc: &str) -> Result> { - if let Ok((content_disposition, content_type, key)) = self.db.search_file_metadata(mxc, 0, 0) { + if let Ok(Metadata { + content_disposition, + content_type, + key, + }) = self.db.search_file_metadata(mxc, 0, 0) + { let mut content = Vec::new(); let path = self.get_media_file(&key); BufReader::new(fs::File::open(path).await?) @@ -249,129 +233,6 @@ impl Service { Ok(deletion_count) } - /// Returns width, height of the thumbnail and whether it should be cropped. - /// Returns None when the server should send the original file. - pub fn thumbnail_properties(&self, width: u32, height: u32) -> Option<(u32, u32, bool)> { - match (width, height) { - (0..=32, 0..=32) => Some((32, 32, true)), - (0..=96, 0..=96) => Some((96, 96, true)), - (0..=320, 0..=240) => Some((320, 240, false)), - (0..=640, 0..=480) => Some((640, 480, false)), - (0..=800, 0..=600) => Some((800, 600, false)), - _ => None, - } - } - - /// Downloads a file's thumbnail. - /// - /// Here's an example on how it works: - /// - /// - Client requests an image with width=567, height=567 - /// - Server rounds that up to (800, 600), so it doesn't have to save too - /// many thumbnails - /// - Server rounds that up again to (958, 600) to fix the aspect ratio - /// (only for width,height>96) - /// - Server creates the thumbnail and sends it to the user - /// - /// For width,height <= 96 the server uses another thumbnailing algorithm - /// which crops the image afterwards. - pub async fn get_thumbnail(&self, mxc: &str, width: u32, height: u32) -> Result> { - let (width, height, crop) = self - .thumbnail_properties(width, height) - .unwrap_or((0, 0, false)); // 0, 0 because that's the original file - - if let Ok((content_disposition, content_type, key)) = self.db.search_file_metadata(mxc, width, height) { - // Using saved thumbnail - let mut content = Vec::new(); - let path = self.get_media_file(&key); - fs::File::open(path) - .await? - .read_to_end(&mut content) - .await?; - - Ok(Some(FileMeta { - content: Some(content), - content_type, - content_disposition, - })) - } else if let Ok((content_disposition, content_type, key)) = self.db.search_file_metadata(mxc, 0, 0) { - // Generate a thumbnail - let mut content = Vec::new(); - let path = self.get_media_file(&key); - fs::File::open(path) - .await? - .read_to_end(&mut content) - .await?; - - if let Ok(image) = image::load_from_memory(&content) { - let original_width = image.width(); - let original_height = image.height(); - if width > original_width || height > original_height { - return Ok(Some(FileMeta { - content: Some(content), - content_type, - content_disposition, - })); - } - - let thumbnail = if crop { - image.resize_to_fill(width, height, FilterType::CatmullRom) - } else { - let (exact_width, exact_height) = { - let ratio = Sat(original_width) * Sat(height); - let nratio = Sat(width) * Sat(original_height); - - let use_width = nratio <= ratio; - let intermediate = if use_width { - Sat(original_height) * Sat(checked!(width / original_width)?) - } else { - Sat(original_width) * Sat(checked!(height / original_height)?) - }; - - if use_width { - (width, intermediate.0) - } else { - (intermediate.0, height) - } - }; - - image.thumbnail_exact(exact_width, exact_height) - }; - - let mut thumbnail_bytes = Vec::new(); - thumbnail.write_to(&mut Cursor::new(&mut thumbnail_bytes), image::ImageFormat::Png)?; - - // Save thumbnail in database so we don't have to generate it again next time - let thumbnail_key = self.db.create_file_metadata( - None, - mxc, - width, - height, - content_disposition.as_deref(), - content_type.as_deref(), - )?; - - let mut f = self.create_media_file(&thumbnail_key).await?; - f.write_all(&thumbnail_bytes).await?; - - Ok(Some(FileMeta { - content: Some(thumbnail_bytes), - content_type, - content_disposition, - })) - } else { - // Couldn't parse file to generate thumbnail, send original - Ok(Some(FileMeta { - content: Some(content), - content_type, - content_disposition, - })) - } - } else { - Ok(None) - } - } - pub async fn get_url_preview(&self, url: &str) -> Option { self.db.get_url_preview(url) } /// TODO: use this? diff --git a/src/service/media/thumbnail.rs b/src/service/media/thumbnail.rs new file mode 100644 index 00000000..50be4ee1 --- /dev/null +++ b/src/service/media/thumbnail.rs @@ -0,0 +1,171 @@ +use std::{io::Cursor, num::Saturating as Sat}; + +use conduit::{checked, Result}; +use image::{imageops::FilterType, DynamicImage}; +use ruma::OwnedUserId; +use tokio::{ + fs, + io::{AsyncReadExt, AsyncWriteExt}, +}; + +use super::{data::Metadata, FileMeta}; + +impl super::Service { + /// Uploads or replaces a file thumbnail. + #[allow(clippy::too_many_arguments)] + pub async fn upload_thumbnail( + &self, sender_user: Option, mxc: &str, content_disposition: Option<&str>, + content_type: Option<&str>, width: u32, height: u32, file: &[u8], + ) -> Result<()> { + let key = if let Some(user) = sender_user { + self.db + .create_file_metadata(Some(user.as_str()), mxc, width, height, content_disposition, content_type)? + } else { + self.db + .create_file_metadata(None, mxc, width, height, content_disposition, content_type)? + }; + + //TODO: Dangling metadata in database if creation fails + let mut f = self.create_media_file(&key).await?; + f.write_all(file).await?; + + Ok(()) + } + + /// Downloads a file's thumbnail. + /// + /// Here's an example on how it works: + /// + /// - Client requests an image with width=567, height=567 + /// - Server rounds that up to (800, 600), so it doesn't have to save too + /// many thumbnails + /// - Server rounds that up again to (958, 600) to fix the aspect ratio + /// (only for width,height>96) + /// - Server creates the thumbnail and sends it to the user + /// + /// For width,height <= 96 the server uses another thumbnailing algorithm + /// which crops the image afterwards. + #[tracing::instrument(skip(self), name = "thumbnail", level = "debug")] + pub async fn get_thumbnail(&self, mxc: &str, width: u32, height: u32) -> Result> { + // 0, 0 because that's the original file + let (width, height, crop) = thumbnail_properties(width, height).unwrap_or((0, 0, false)); + + if let Ok(metadata) = self.db.search_file_metadata(mxc, width, height) { + self.get_thumbnail_saved(metadata).await + } else if let Ok(metadata) = self.db.search_file_metadata(mxc, 0, 0) { + self.get_thumbnail_generate(mxc, width, height, crop, metadata) + .await + } else { + Ok(None) + } + } + + /// Using saved thumbnail + #[tracing::instrument(skip(self), name = "saved", level = "debug")] + async fn get_thumbnail_saved(&self, data: Metadata) -> Result> { + let mut content = Vec::new(); + let path = self.get_media_file(&data.key); + fs::File::open(path) + .await? + .read_to_end(&mut content) + .await?; + + Ok(Some(into_filemeta(data, content))) + } + + /// Generate a thumbnail + #[tracing::instrument(skip(self), name = "generate", level = "debug")] + async fn get_thumbnail_generate( + &self, mxc: &str, width: u32, height: u32, crop: bool, data: Metadata, + ) -> Result> { + let mut content = Vec::new(); + let path = self.get_media_file(&data.key); + fs::File::open(path) + .await? + .read_to_end(&mut content) + .await?; + + let Ok(image) = image::load_from_memory(&content) else { + // Couldn't parse file to generate thumbnail, send original + return Ok(Some(into_filemeta(data, content))); + }; + + if width > image.width() || height > image.height() { + return Ok(Some(into_filemeta(data, content))); + } + + let mut thumbnail_bytes = Vec::new(); + let thumbnail = thumbnail_generate(&image, width, height, crop)?; + thumbnail.write_to(&mut Cursor::new(&mut thumbnail_bytes), image::ImageFormat::Png)?; + + // Save thumbnail in database so we don't have to generate it again next time + let thumbnail_key = self.db.create_file_metadata( + None, + mxc, + width, + height, + data.content_disposition.as_deref(), + data.content_type.as_deref(), + )?; + + let mut f = self.create_media_file(&thumbnail_key).await?; + f.write_all(&thumbnail_bytes).await?; + + Ok(Some(into_filemeta(data, thumbnail_bytes))) + } +} + +fn thumbnail_generate(image: &DynamicImage, width: u32, height: u32, crop: bool) -> Result { + let thumbnail = if crop { + image.resize_to_fill(width, height, FilterType::CatmullRom) + } else { + let (exact_width, exact_height) = thumbnail_dimension(image, width, height)?; + image.thumbnail_exact(exact_width, exact_height) + }; + + Ok(thumbnail) +} + +fn thumbnail_dimension(image: &DynamicImage, width: u32, height: u32) -> Result<(u32, u32)> { + let original_width = image.width(); + let original_height = image.height(); + + let ratio = Sat(original_width) * Sat(height); + let nratio = Sat(width) * Sat(original_height); + let use_width = nratio <= ratio; + + let intermediate = if use_width { + Sat(original_height) * Sat(checked!(width / original_width)?) + } else { + Sat(original_width) * Sat(checked!(height / original_height)?) + }; + + let dims = if use_width { + (width, intermediate.0) + } else { + (intermediate.0, height) + }; + + Ok(dims) +} + +/// Returns width, height of the thumbnail and whether it should be cropped. +/// Returns None when the server should send the original file. +fn thumbnail_properties(width: u32, height: u32) -> Option<(u32, u32, bool)> { + match (width, height) { + (0..=32, 0..=32) => Some((32, 32, true)), + (0..=96, 0..=96) => Some((96, 96, true)), + (0..=320, 0..=240) => Some((320, 240, false)), + (0..=640, 0..=480) => Some((640, 480, false)), + (0..=800, 0..=600) => Some((800, 600, false)), + _ => None, + } +} + +fn into_filemeta(data: Metadata, content: Vec) -> FileMeta { + FileMeta { + content: Some(content), + content_type: data.content_type, + content_disposition: data.content_disposition, + } +} From d67f19a55d98a535622f98015fab434ee2ec0278 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 15 Jul 2024 04:19:43 +0000 Subject: [PATCH 153/158] cleanup some error callsites Signed-off-by: Jason Volk --- src/api/router/args.rs | 30 ++++-------- src/api/router/request.rs | 11 +++-- src/api/server/send.rs | 29 ++++-------- src/core/error/err.rs | 1 + src/core/error/mod.rs | 2 +- src/core/utils/math.rs | 8 ++-- src/service/admin/mod.rs | 8 ++-- src/service/media/mod.rs | 52 +++++++------------- src/service/users/data.rs | 99 +++++++++++++++++---------------------- 9 files changed, 96 insertions(+), 144 deletions(-) diff --git a/src/api/router/args.rs b/src/api/router/args.rs index b54a6f80..776ce4f4 100644 --- a/src/api/router/args.rs +++ b/src/api/router/args.rs @@ -2,11 +2,8 @@ use std::{mem, ops::Deref}; use axum::{async_trait, body::Body, extract::FromRequest}; use bytes::{BufMut, BytesMut}; -use conduit::{debug, debug_warn, trace, warn, Error, Result}; -use ruma::{ - api::{client::error::ErrorKind, IncomingRequest}, - CanonicalJsonValue, OwnedDeviceId, OwnedServerName, OwnedUserId, UserId, -}; +use conduit::{debug, err, trace, Error, Result}; +use ruma::{api::IncomingRequest, CanonicalJsonValue, OwnedDeviceId, OwnedServerName, OwnedUserId, UserId}; use super::{auth, auth::Auth, request, request::Request}; use crate::{service::appservice::RegistrationInfo, services}; @@ -103,21 +100,14 @@ where let mut http_request = hyper::Request::builder() .uri(request.parts.uri.clone()) .method(request.parts.method.clone()); - *http_request.headers_mut().unwrap() = request.parts.headers.clone(); - let http_request = http_request.body(body).unwrap(); - debug!( - "{:?} {:?} {:?}", - http_request.method(), - http_request.uri(), - http_request.headers() - ); + *http_request.headers_mut().expect("mutable http headers") = request.parts.headers.clone(); + let http_request = http_request.body(body).expect("http request body"); - trace!("{:?} {:?} {:?}", http_request.method(), http_request.uri(), json_body); - let body = T::try_from_http_request(http_request, &request.path).map_err(|e| { - warn!("try_from_http_request failed: {e:?}",); - debug_warn!("JSON body: {:?}", json_body); - Error::BadRequest(ErrorKind::BadJson, "Failed to deserialize request.") - })?; + let headers = http_request.headers(); + let method = http_request.method(); + let uri = http_request.uri(); + debug!("{method:?} {uri:?} {headers:?}"); + trace!("{method:?} {uri:?} {json_body:?}"); - Ok(body) + T::try_from_http_request(http_request, &request.path).map_err(|e| err!(Request(BadJson(debug_warn!("{e}"))))) } diff --git a/src/api/router/request.rs b/src/api/router/request.rs index 56c76619..bed8d057 100644 --- a/src/api/router/request.rs +++ b/src/api/router/request.rs @@ -2,11 +2,11 @@ use std::str; use axum::{extract::Path, RequestExt, RequestPartsExt}; use bytes::Bytes; +use conduit::err; use http::request::Parts; -use ruma::api::client::error::ErrorKind; use serde::Deserialize; -use crate::{services, Error, Result}; +use crate::{services, Result}; #[derive(Deserialize)] pub(super) struct QueryParams { @@ -26,14 +26,15 @@ pub(super) async fn from(request: hyper::Request) -> Result> = parts.extract().await?; - let query = serde_html_form::from_str(parts.uri.query().unwrap_or_default()) - .map_err(|_| Error::BadRequest(ErrorKind::Unknown, "Failed to read query parameters"))?; + let query = parts.uri.query().unwrap_or_default(); + let query = + serde_html_form::from_str(query).map_err(|e| err!(Request(Unknown("Failed to read query parameters: {e}"))))?; let max_body_size = services().globals.config.max_request_size; let body = axum::body::to_bytes(body, max_body_size) .await - .map_err(|_| Error::BadRequest(ErrorKind::TooLarge, "Request body too large"))?; + .map_err(|e| err!(Request(TooLarge("Request body too large: {e}"))))?; Ok(Request { path, diff --git a/src/api/server/send.rs b/src/api/server/send.rs index 122f564f..31f8e73b 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -1,7 +1,7 @@ use std::{collections::BTreeMap, net::IpAddr, time::Instant}; use axum_client_ip::InsecureClientIp; -use conduit::debug_warn; +use conduit::{debug, debug_warn, err, trace, warn, Err}; use ruma::{ api::{ client::error::ErrorKind, @@ -18,7 +18,6 @@ use ruma::{ OwnedEventId, ServerName, }; use tokio::sync::RwLock; -use tracing::{debug, error, trace, warn}; use crate::{ service::rooms::event_handler::parse_incoming_pdu, @@ -39,24 +38,17 @@ pub(crate) async fn send_transaction_message_route( let origin = body.origin.as_ref().expect("server is authenticated"); if *origin != body.body.origin { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Not allowed to send transactions on behalf of other servers", - )); + return Err!(Request(Forbidden( + "Not allowed to send transactions on behalf of other servers" + ))); } if body.pdus.len() > 50_usize { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Not allowed to send more than 50 PDUs in one transaction", - )); + return Err!(Request(Forbidden("Not allowed to send more than 50 PDUs in one transaction"))); } if body.edus.len() > 100_usize { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Not allowed to send more than 100 EDUs in one transaction", - )); + return Err!(Request(Forbidden("Not allowed to send more than 100 EDUs in one transaction"))); } let txn_start_time = Instant::now(); @@ -392,10 +384,9 @@ async fn handle_edu_direct_to_device( target_user_id, target_device_id, &ev_type.to_string(), - event.deserialize_as().map_err(|e| { - error!("To-Device event is invalid: {event:?} {e}"); - Error::BadRequest(ErrorKind::InvalidParam, "Event is invalid") - })?, + event + .deserialize_as() + .map_err(|e| err!(Request(InvalidParam(error!("To-Device event is invalid: {e}")))))?, )?; }, @@ -408,7 +399,7 @@ async fn handle_edu_direct_to_device( &ev_type.to_string(), event .deserialize_as() - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Event is invalid"))?, + .map_err(|e| err!(Request(InvalidParam("Event is invalid: {e}"))))?, )?; } }, diff --git a/src/core/error/err.rs b/src/core/error/err.rs index 66634f2b..55e38f18 100644 --- a/src/core/error/err.rs +++ b/src/core/error/err.rs @@ -31,6 +31,7 @@ //! Err(Database(error!("problem with db: {msg}")))` logs the error at the //! callsite and then returns the error with the same string. Caller has the //! option of replacing `error!` with `debug_error!`. + #[macro_export] macro_rules! Err { ($($args:tt)*) => { diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index 1f0d56c1..9439261e 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -78,7 +78,7 @@ pub enum Error { // conduwuit #[error("Arithmetic operation failed: {0}")] - Arithmetic(&'static str), + Arithmetic(Cow<'static, str>), #[error("There was a problem with the '{0}' directive in your configuration: {1}")] Config(&'static str, Cow<'static, str>), #[error("{0}")] diff --git a/src/core/utils/math.rs b/src/core/utils/math.rs index 155721e7..96ac6dc2 100644 --- a/src/core/utils/math.rs +++ b/src/core/utils/math.rs @@ -2,14 +2,14 @@ use std::{cmp, time::Duration}; pub use checked_ops::checked_ops; -use crate::{Error, Result}; +use crate::{Err, Error, Result}; /// Checked arithmetic expression. Returns a Result #[macro_export] macro_rules! checked { ($($input:tt)*) => { $crate::utils::math::checked_ops!($($input)*) - .ok_or_else(|| $crate::Error::Arithmetic("operation overflowed or result invalid")) + .ok_or_else(|| $crate::err!(Arithmetic("operation overflowed or result invalid"))) } } @@ -21,7 +21,7 @@ macro_rules! validated { ($($input:tt)*) => { //#[allow(clippy::arithmetic_side_effects)] { //Some($($input)*) - // .ok_or_else(|| $crate::Error::Arithmetic("this error should never been seen")) + // .ok_or_else(|| $crate::err!(Arithmetic("this error should never been seen"))) //} //NOTE: remove me when stmt_expr_attributes is stable @@ -57,7 +57,7 @@ pub fn continue_exponential_backoff(min: Duration, max: Duration, elapsed: Durat #[allow(clippy::as_conversions)] pub fn usize_from_f64(val: f64) -> Result { if val < 0.0 { - return Err(Error::Arithmetic("Converting negative float to unsigned integer")); + return Err!(Arithmetic("Converting negative float to unsigned integer")); } //SAFETY: diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index 41019cd1..f63ebf09 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -9,7 +9,7 @@ use std::{ }; use async_trait::async_trait; -use conduit::{debug, error, Error, Result}; +use conduit::{debug, error, error::default_log, Error, Result}; pub use create::create_admin_room; pub use grant::make_user_admin; use loole::{Receiver, Sender}; @@ -239,9 +239,9 @@ async fn respond_to_room(content: RoomMessageEventContent, room_id: &RoomId, use .build_and_append_pdu(response_pdu, user_id, room_id, &state_lock) .await { - if let Err(e) = handle_response_error(e, room_id, user_id, &state_lock).await { - error!("{e}"); - } + handle_response_error(e, room_id, user_id, &state_lock) + .await + .unwrap_or_else(default_log); } } diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index c969d1e8..1638235b 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -6,7 +6,7 @@ use std::{collections::HashMap, path::PathBuf, sync::Arc, time::SystemTime}; use async_trait::async_trait; use base64::{engine::general_purpose, Engine as _}; -use conduit::{debug, debug_error, err, error, utils, Result, Server}; +use conduit::{debug, debug_error, err, error, utils, Err, Result, Server}; use data::{Data, Metadata}; use ruma::{OwnedMxcUri, OwnedUserId}; use serde::Serialize; @@ -100,10 +100,9 @@ impl Service { Ok(()) } else { - error!("Failed to find any media keys for MXC \"{mxc}\" in our database (MXC does not exist)"); - Err(Error::bad_database( - "Failed to find any media keys for the provided MXC in our database (MXC does not exist)", - )) + Err!(Database(error!( + "Failed to find any media keys for MXC {mxc:?} in our database." + ))) } } @@ -137,23 +136,16 @@ impl Service { let all_keys = self.db.get_all_media_keys(); let user_duration: SystemTime = match cyborgtime::parse_duration(&time) { - Ok(duration) => { - debug!("Parsed duration: {:?}", duration); - debug!("System time now: {:?}", SystemTime::now()); - SystemTime::now().checked_sub(duration).ok_or_else(|| { - Error::bad_database("Duration specified is not valid against the current system time") - })? - }, - Err(e) => { - error!("Failed to parse user-specified time duration: {}", e); - return Err(Error::bad_database("Failed to parse user-specified time duration.")); - }, + Err(e) => return Err!(Database(error!("Failed to parse specified time duration: {e}"))), + Ok(duration) => SystemTime::now() + .checked_sub(duration) + .ok_or(err!(Arithmetic("Duration {duration:?} is too large")))?, }; let mut remote_mxcs: Vec = vec![]; for key in all_keys { - debug!("Full MXC key from database: {:?}", key); + debug!("Full MXC key from database: {key:?}"); // we need to get the MXC URL from the first part of the key (the first 0xff / // 255 push). this is all necessary because of conduit using magic keys for @@ -162,24 +154,19 @@ impl Service { let mxc = parts .next() .map(|bytes| { - utils::string_from_bytes(bytes).map_err(|e| { - error!("Failed to parse MXC unicode bytes from our database: {}", e); - Error::bad_database("Failed to parse MXC unicode bytes from our database") - }) + utils::string_from_bytes(bytes) + .map_err(|e| err!(Database(error!("Failed to parse MXC unicode bytes from our database: {e}")))) }) .transpose()?; let Some(mxc_s) = mxc else { - return Err(Error::bad_database( - "Parsed MXC URL unicode bytes from database but still is None", - )); + return Err!(Database("Parsed MXC URL unicode bytes from database but still is None")); }; - debug!("Parsed MXC key to URL: {}", mxc_s); - + debug!("Parsed MXC key to URL: {mxc_s}"); let mxc = OwnedMxcUri::from(mxc_s); if mxc.server_name() == Ok(services().globals.server_name()) { - debug!("Ignoring local media MXC: {}", mxc); + debug!("Ignoring local media MXC: {mxc}"); // ignore our own MXC URLs as this would be local media. continue; } @@ -198,14 +185,14 @@ impl Service { }, Err(err) => { if force { - error!("Could not delete MXC path {:?}: {:?}. Skipping...", path, err); + error!("Could not delete MXC path {path:?}: {err:?}. Skipping..."); continue; } return Err(err.into()); }, }; - debug!("File created at: {:?}", file_created_at); + debug!("File created at: {file_created_at:?}"); if file_created_at <= user_duration { debug!("File is within user duration, pushing to list of file paths and keys to delete."); remote_mxcs.push(mxc.to_string()); @@ -215,15 +202,12 @@ impl Service { debug!( "Finished going through all our media in database for eligible keys to delete, checking if these are empty" ); - if remote_mxcs.is_empty() { - return Err(Error::bad_database("Did not found any eligible MXCs to delete.")); + return Err!(Database("Did not found any eligible MXCs to delete.")); } - debug!("Deleting media now in the past \"{:?}\".", user_duration); - + debug!("Deleting media now in the past {user_duration:?}."); let mut deletion_count: usize = 0; - for mxc in remote_mxcs { debug!("Deleting MXC {mxc} from database and filesystem"); self.delete(&mxc).await?; diff --git a/src/service/users/data.rs b/src/service/users/data.rs index 302eae9d..5546adb1 100644 --- a/src/service/users/data.rs +++ b/src/service/users/data.rs @@ -1,6 +1,6 @@ use std::{collections::BTreeMap, mem::size_of, sync::Arc}; -use conduit::{debug_info, utils, warn, Error, Result}; +use conduit::{debug_info, err, utils, warn, Err, Error, Result}; use database::{Database, Map}; use ruma::{ api::client::{device::Device, error::ErrorKind, filter::FilterDefinition}, @@ -87,19 +87,19 @@ impl Data { let mut parts = bytes.split(|&b| b == 0xFF); let user_bytes = parts .next() - .ok_or_else(|| Error::bad_database("User ID in token_userdeviceid is invalid."))?; + .ok_or_else(|| err!(Database("User ID in token_userdeviceid is invalid.")))?; let device_bytes = parts .next() - .ok_or_else(|| Error::bad_database("Device ID in token_userdeviceid is invalid."))?; + .ok_or_else(|| err!(Database("Device ID in token_userdeviceid is invalid.")))?; Ok(Some(( UserId::parse( utils::string_from_bytes(user_bytes) - .map_err(|_| Error::bad_database("User ID in token_userdeviceid is invalid unicode."))?, + .map_err(|e| err!(Database("User ID in token_userdeviceid is invalid unicode. {e}")))?, ) - .map_err(|_| Error::bad_database("User ID in token_userdeviceid is invalid."))?, + .map_err(|e| err!(Database("User ID in token_userdeviceid is invalid. {e}")))?, utils::string_from_bytes(device_bytes) - .map_err(|_| Error::bad_database("Device ID in token_userdeviceid is invalid."))?, + .map_err(|e| err!(Database("Device ID in token_userdeviceid is invalid. {e}")))?, ))) }) } @@ -109,9 +109,9 @@ impl Data { Box::new(self.userid_password.iter().map(|(bytes, _)| { UserId::parse( utils::string_from_bytes(&bytes) - .map_err(|_| Error::bad_database("User ID in userid_password is invalid unicode."))?, + .map_err(|e| err!(Database("User ID in userid_password is invalid unicode. {e}")))?, ) - .map_err(|_| Error::bad_database("User ID in userid_password is invalid.")) + .map_err(|e| err!(Database("User ID in userid_password is invalid. {e}"))) })) } @@ -165,7 +165,7 @@ impl Data { .map_or(Ok(None), |bytes| { Ok(Some( utils::string_from_bytes(&bytes) - .map_err(|_| Error::bad_database("Displayname in db is invalid."))?, + .map_err(|e| err!(Database("Displayname in db is invalid. {e}")))?, )) }) } @@ -188,10 +188,8 @@ impl Data { self.userid_avatarurl .get(user_id.as_bytes())? .map(|bytes| { - let s_bytes = utils::string_from_bytes(&bytes).map_err(|e| { - warn!("Avatar URL in db is invalid: {}", e); - Error::bad_database("Avatar URL in db is invalid.") - })?; + let s_bytes = utils::string_from_bytes(&bytes) + .map_err(|e| err!(Database(warn!("Avatar URL in db is invalid: {e}"))))?; let mxc_uri: OwnedMxcUri = s_bytes.into(); Ok(mxc_uri) }) @@ -215,10 +213,7 @@ impl Data { self.userid_blurhash .get(user_id.as_bytes())? .map(|bytes| { - let s = utils::string_from_bytes(&bytes) - .map_err(|_| Error::bad_database("Avatar URL in db is invalid."))?; - - Ok(s) + utils::string_from_bytes(&bytes).map_err(|e| err!(Database("Avatar URL in db is invalid. {e}"))) }) .transpose() } @@ -314,9 +309,9 @@ impl Data { bytes .rsplit(|&b| b == 0xFF) .next() - .ok_or_else(|| Error::bad_database("UserDevice ID in db is invalid."))?, + .ok_or_else(|| err!(Database("UserDevice ID in db is invalid.")))?, ) - .map_err(|_| Error::bad_database("Device ID in userdeviceid_metadata is invalid."))? + .map_err(|e| err!(Database("Device ID in userdeviceid_metadata is invalid. {e}")))? .into()) }), ) @@ -330,13 +325,9 @@ impl Data { // should not be None, but we shouldn't assert either lol... if self.userdeviceid_metadata.get(&userdeviceid)?.is_none() { - warn!( - "Called set_token for a non-existent user \"{}\" and/or device ID \"{}\" with no metadata in database", - user_id, device_id - ); - return Err(Error::bad_database( - "User does not exist or device ID has no metadata in database.", - )); + return Err!(Database(error!( + "User {user_id:?} does not exist or device ID {device_id:?} has no metadata." + ))); } // Remove old token @@ -366,14 +357,9 @@ impl Data { // Only existing devices should be able to call this, but we shouldn't assert // either... if self.userdeviceid_metadata.get(&key)?.is_none() { - warn!( - "Called add_one_time_key for a non-existent user \"{}\" and/or device ID \"{}\" with no metadata in \ - database", - user_id, device_id - ); - return Err(Error::bad_database( - "User does not exist or device ID has no metadata in database.", - )); + return Err!(Database(error!( + "User {user_id:?} does not exist or device ID {device_id:?} has no metadata." + ))); } key.push(0xFF); @@ -401,7 +387,7 @@ impl Data { .get(user_id.as_bytes())? .map_or(Ok(0), |bytes| { utils::u64_from_bytes(&bytes) - .map_err(|_| Error::bad_database("Count in roomid_lastroomactiveupdate is invalid.")) + .map_err(|e| err!(Database("Count in roomid_lastroomactiveupdate is invalid. {e}"))) }) } @@ -429,11 +415,10 @@ impl Data { serde_json::from_slice( key.rsplit(|&b| b == 0xFF) .next() - .ok_or_else(|| Error::bad_database("OneTimeKeyId in db is invalid."))?, + .ok_or_else(|| err!(Database("OneTimeKeyId in db is invalid.")))?, ) - .map_err(|_| Error::bad_database("OneTimeKeyId in db is invalid."))?, - serde_json::from_slice(&value) - .map_err(|_| Error::bad_database("OneTimeKeys in db are invalid."))?, + .map_err(|e| err!(Database("OneTimeKeyId in db is invalid. {e}")))?, + serde_json::from_slice(&value).map_err(|e| err!(Database("OneTimeKeys in db are invalid. {e}")))?, )) }) .transpose() @@ -457,9 +442,9 @@ impl Data { bytes .rsplit(|&b| b == 0xFF) .next() - .ok_or_else(|| Error::bad_database("OneTimeKey ID in db is invalid."))?, + .ok_or_else(|| err!(Database("OneTimeKey ID in db is invalid.")))?, ) - .map_err(|_| Error::bad_database("DeviceKeyId in db is invalid."))? + .map_err(|e| err!(Database("DeviceKeyId in db is invalid. {e}")))? .algorithm(), ) }) { @@ -581,19 +566,19 @@ impl Data { .get(&key)? .ok_or(Error::BadRequest(ErrorKind::InvalidParam, "Tried to sign nonexistent key."))?, ) - .map_err(|_| Error::bad_database("key in keyid_key is invalid."))?; + .map_err(|e| err!(Database("key in keyid_key is invalid. {e}")))?; let signatures = cross_signing_key .get_mut("signatures") - .ok_or_else(|| Error::bad_database("key in keyid_key has no signatures field."))? + .ok_or_else(|| err!(Database("key in keyid_key has no signatures field.")))? .as_object_mut() - .ok_or_else(|| Error::bad_database("key in keyid_key has invalid signatures field."))? + .ok_or_else(|| err!(Database("key in keyid_key has invalid signatures field.")))? .entry(sender_id.to_string()) .or_insert_with(|| serde_json::Map::new().into()); signatures .as_object_mut() - .ok_or_else(|| Error::bad_database("signatures in keyid_key for a user is invalid."))? + .ok_or_else(|| err!(Database("signatures in keyid_key for a user is invalid.")))? .insert(signature.0, signature.1.into()); self.keyid_key.insert( @@ -640,7 +625,7 @@ impl Data { Error::bad_database("User ID in devicekeychangeid_userid is invalid unicode.") })?, ) - .map_err(|_| Error::bad_database("User ID in devicekeychangeid_userid is invalid.")) + .map_err(|e| err!(Database("User ID in devicekeychangeid_userid is invalid. {e}"))) }), ) } @@ -685,7 +670,7 @@ impl Data { self.keyid_key.get(&key)?.map_or(Ok(None), |bytes| { Ok(Some( - serde_json::from_slice(&bytes).map_err(|_| Error::bad_database("DeviceKeys in db are invalid."))?, + serde_json::from_slice(&bytes).map_err(|e| err!(Database("DeviceKeys in db are invalid. {e}")))?, )) }) } @@ -719,7 +704,7 @@ impl Data { ) -> Result>> { self.keyid_key.get(key)?.map_or(Ok(None), |bytes| { let mut cross_signing_key = serde_json::from_slice::(&bytes) - .map_err(|_| Error::bad_database("CrossSigningKey in db is invalid."))?; + .map_err(|e| err!(Database("CrossSigningKey in db is invalid. {e}")))?; clean_signatures(&mut cross_signing_key, sender_user, user_id, allowed_signatures)?; Ok(Some(Raw::from_json( @@ -751,7 +736,7 @@ impl Data { self.keyid_key.get(&key)?.map_or(Ok(None), |bytes| { Ok(Some( serde_json::from_slice(&bytes) - .map_err(|_| Error::bad_database("CrossSigningKey in db is invalid."))?, + .map_err(|e| err!(Database("CrossSigningKey in db is invalid. {e}")))?, )) }) }) @@ -792,7 +777,7 @@ impl Data { for (_, value) in self.todeviceid_events.scan_prefix(prefix) { events.push( serde_json::from_slice(&value) - .map_err(|_| Error::bad_database("Event in todeviceid_events is invalid."))?, + .map_err(|e| err!(Database("Event in todeviceid_events is invalid. {e}")))?, ); } @@ -816,7 +801,7 @@ impl Data { Ok::<_, Error>(( key.clone(), utils::u64_from_bytes(&key[key.len().saturating_sub(size_of::())..key.len()]) - .map_err(|_| Error::bad_database("ToDeviceId has invalid count bytes."))?, + .map_err(|e| err!(Database("ToDeviceId has invalid count bytes. {e}")))?, )) }) .filter_map(Result::ok) @@ -877,7 +862,7 @@ impl Data { .get(user_id.as_bytes())? .map_or(Ok(None), |bytes| { utils::u64_from_bytes(&bytes) - .map_err(|_| Error::bad_database("Invalid devicelistversion in db.")) + .map_err(|e| err!(Database("Invalid devicelistversion in db. {e}"))) .map(Some) }) } @@ -893,7 +878,7 @@ impl Data { .scan_prefix(key) .map(|(_, bytes)| { serde_json::from_slice::(&bytes) - .map_err(|_| Error::bad_database("Device in userdeviceid_metadata is invalid.")) + .map_err(|e| err!(Database("Device in userdeviceid_metadata is invalid. {e}"))) }), ) } @@ -920,7 +905,7 @@ impl Data { let raw = self.userfilterid_filter.get(&key)?; if let Some(raw) = raw { - serde_json::from_slice(&raw).map_err(|_| Error::bad_database("Invalid filter event in db.")) + serde_json::from_slice(&raw).map_err(|e| err!(Database("Invalid filter event in db. {e}"))) } else { Ok(None) } @@ -954,7 +939,7 @@ impl Data { let expires_at = u64::from_be_bytes( expires_at_bytes .try_into() - .map_err(|_| Error::bad_database("expires_at in openid_userid is invalid u64."))?, + .map_err(|e| err!(Database("expires_at in openid_userid is invalid u64. {e}")))?, ); if expires_at < utils::millis_since_unix_epoch() { @@ -966,9 +951,9 @@ impl Data { UserId::parse( utils::string_from_bytes(user_bytes) - .map_err(|_| Error::bad_database("User ID in openid_userid is invalid unicode."))?, + .map_err(|e| err!(Database("User ID in openid_userid is invalid unicode. {e}")))?, ) - .map_err(|_| Error::bad_database("User ID in openid_userid is invalid.")) + .map_err(|e| err!(Database("User ID in openid_userid is invalid. {e}"))) } } From 4430e4dee037c02ba02a3e5f463edbc0f66d800a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 15 Jul 2024 10:37:16 +0000 Subject: [PATCH 154/158] de-global some services() Signed-off-by: Jason Volk --- src/api/server/send.rs | 89 +++++++++++++++++++++++------------------- 1 file changed, 49 insertions(+), 40 deletions(-) diff --git a/src/api/server/send.rs b/src/api/server/send.rs index 31f8e73b..7c699e95 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -1,5 +1,6 @@ use std::{collections::BTreeMap, net::IpAddr, time::Instant}; +use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduit::{debug, debug_warn, err, trace, warn, Err}; use ruma::{ @@ -21,7 +22,7 @@ use tokio::sync::RwLock; use crate::{ service::rooms::event_handler::parse_incoming_pdu, - services, + services::Services, utils::{self}, Error, Result, Ruma, }; @@ -33,7 +34,8 @@ type ResolvedMap = BTreeMap>; /// Push EDUs and PDUs to this server. #[tracing::instrument(skip_all, fields(%client), name = "send")] pub(crate) async fn send_transaction_message_route( - InsecureClientIp(client): InsecureClientIp, body: Ruma, + State(services): State<&Services>, InsecureClientIp(client): InsecureClientIp, + body: Ruma, ) -> Result { let origin = body.origin.as_ref().expect("server is authenticated"); @@ -61,8 +63,8 @@ pub(crate) async fn send_transaction_message_route( "Starting txn", ); - let resolved_map = handle_pdus(&client, &body, origin, &txn_start_time).await?; - handle_edus(&client, &body, origin).await?; + let resolved_map = handle_pdus(services, &client, &body, origin, &txn_start_time).await?; + handle_edus(services, &client, &body, origin).await?; debug!( pdus = ?body.pdus.len(), @@ -82,7 +84,8 @@ pub(crate) async fn send_transaction_message_route( } async fn handle_pdus( - _client: &IpAddr, body: &Ruma, origin: &ServerName, txn_start_time: &Instant, + services: &Services, _client: &IpAddr, body: &Ruma, origin: &ServerName, + txn_start_time: &Instant, ) -> Result { let mut parsed_pdus = Vec::with_capacity(body.pdus.len()); for pdu in &body.pdus { @@ -102,7 +105,7 @@ async fn handle_pdus( // corresponding signing keys let pub_key_map = RwLock::new(BTreeMap::new()); if !parsed_pdus.is_empty() { - services() + services .rooms .event_handler .fetch_required_signing_keys(parsed_pdus.iter().map(|(_event_id, event, _room_id)| event), &pub_key_map) @@ -118,7 +121,7 @@ async fn handle_pdus( let mut resolved_map = BTreeMap::new(); for (event_id, value, room_id) in parsed_pdus { let pdu_start_time = Instant::now(); - let mutex_lock = services() + let mutex_lock = services .rooms .event_handler .mutex_federation @@ -126,7 +129,7 @@ async fn handle_pdus( .await; resolved_map.insert( event_id.clone(), - services() + services .rooms .event_handler .handle_incoming_pdu(origin, &room_id, &event_id, value, true, &pub_key_map) @@ -154,7 +157,7 @@ async fn handle_pdus( } async fn handle_edus( - client: &IpAddr, body: &Ruma, origin: &ServerName, + services: &Services, client: &IpAddr, body: &Ruma, origin: &ServerName, ) -> Result<()> { for edu in body .edus @@ -162,12 +165,12 @@ async fn handle_edus( .filter_map(|edu| serde_json::from_str::(edu.json().get()).ok()) { match edu { - Edu::Presence(presence) => handle_edu_presence(client, origin, presence).await?, - Edu::Receipt(receipt) => handle_edu_receipt(client, origin, receipt).await?, - Edu::Typing(typing) => handle_edu_typing(client, origin, typing).await?, - Edu::DeviceListUpdate(content) => handle_edu_device_list_update(client, origin, content).await?, - Edu::DirectToDevice(content) => handle_edu_direct_to_device(client, origin, content).await?, - Edu::SigningKeyUpdate(content) => handle_edu_signing_key_update(client, origin, content).await?, + Edu::Presence(presence) => handle_edu_presence(services, client, origin, presence).await?, + Edu::Receipt(receipt) => handle_edu_receipt(services, client, origin, receipt).await?, + Edu::Typing(typing) => handle_edu_typing(services, client, origin, typing).await?, + Edu::DeviceListUpdate(content) => handle_edu_device_list_update(services, client, origin, content).await?, + Edu::DirectToDevice(content) => handle_edu_direct_to_device(services, client, origin, content).await?, + Edu::SigningKeyUpdate(content) => handle_edu_signing_key_update(services, client, origin, content).await?, Edu::_Custom(ref _custom) => { debug_warn!(?body.edus, "received custom/unknown EDU"); }, @@ -177,8 +180,10 @@ async fn handle_edus( Ok(()) } -async fn handle_edu_presence(_client: &IpAddr, origin: &ServerName, presence: PresenceContent) -> Result<()> { - if !services().globals.allow_incoming_presence() { +async fn handle_edu_presence( + services: &Services, _client: &IpAddr, origin: &ServerName, presence: PresenceContent, +) -> Result<()> { + if !services.globals.allow_incoming_presence() { return Ok(()); } @@ -191,7 +196,7 @@ async fn handle_edu_presence(_client: &IpAddr, origin: &ServerName, presence: Pr continue; } - services().presence.set_presence( + services.presence.set_presence( &update.user_id, &update.presence, Some(update.currently_active), @@ -203,13 +208,15 @@ async fn handle_edu_presence(_client: &IpAddr, origin: &ServerName, presence: Pr Ok(()) } -async fn handle_edu_receipt(_client: &IpAddr, origin: &ServerName, receipt: ReceiptContent) -> Result<()> { - if !services().globals.allow_incoming_read_receipts() { +async fn handle_edu_receipt( + services: &Services, _client: &IpAddr, origin: &ServerName, receipt: ReceiptContent, +) -> Result<()> { + if !services.globals.allow_incoming_read_receipts() { return Ok(()); } for (room_id, room_updates) in receipt.receipts { - if services() + if services .rooms .event_handler .acl_check(origin, &room_id) @@ -231,7 +238,7 @@ async fn handle_edu_receipt(_client: &IpAddr, origin: &ServerName, receipt: Rece continue; } - if services() + if services .rooms .state_cache .room_members(&room_id) @@ -247,7 +254,7 @@ async fn handle_edu_receipt(_client: &IpAddr, origin: &ServerName, receipt: Rece room_id: room_id.clone(), }; - services() + services .rooms .read_receipt .readreceipt_update(&user_id, &room_id, &event)?; @@ -265,8 +272,10 @@ async fn handle_edu_receipt(_client: &IpAddr, origin: &ServerName, receipt: Rece Ok(()) } -async fn handle_edu_typing(_client: &IpAddr, origin: &ServerName, typing: TypingContent) -> Result<()> { - if !services().globals.config.allow_incoming_typing { +async fn handle_edu_typing( + services: &Services, _client: &IpAddr, origin: &ServerName, typing: TypingContent, +) -> Result<()> { + if !services.globals.config.allow_incoming_typing { return Ok(()); } @@ -278,7 +287,7 @@ async fn handle_edu_typing(_client: &IpAddr, origin: &ServerName, typing: Typing return Ok(()); } - if services() + if services .rooms .event_handler .acl_check(typing.user_id.server_name(), &typing.room_id) @@ -291,26 +300,26 @@ async fn handle_edu_typing(_client: &IpAddr, origin: &ServerName, typing: Typing return Ok(()); } - if services() + if services .rooms .state_cache .is_joined(&typing.user_id, &typing.room_id)? { if typing.typing { let timeout = utils::millis_since_unix_epoch().saturating_add( - services() + services .globals .config .typing_federation_timeout_s .saturating_mul(1000), ); - services() + services .rooms .typing .typing_add(&typing.user_id, &typing.room_id, timeout) .await?; } else { - services() + services .rooms .typing .typing_remove(&typing.user_id, &typing.room_id) @@ -328,7 +337,7 @@ async fn handle_edu_typing(_client: &IpAddr, origin: &ServerName, typing: Typing } async fn handle_edu_device_list_update( - _client: &IpAddr, origin: &ServerName, content: DeviceListUpdateContent, + services: &Services, _client: &IpAddr, origin: &ServerName, content: DeviceListUpdateContent, ) -> Result<()> { let DeviceListUpdateContent { user_id, @@ -343,13 +352,13 @@ async fn handle_edu_device_list_update( return Ok(()); } - services().users.mark_device_key_update(&user_id)?; + services.users.mark_device_key_update(&user_id)?; Ok(()) } async fn handle_edu_direct_to_device( - _client: &IpAddr, origin: &ServerName, content: DirectDeviceContent, + services: &Services, _client: &IpAddr, origin: &ServerName, content: DirectDeviceContent, ) -> Result<()> { let DirectDeviceContent { sender, @@ -367,7 +376,7 @@ async fn handle_edu_direct_to_device( } // Check if this is a new transaction id - if services() + if services .transaction_ids .existing_txnid(&sender, None, &message_id)? .is_some() @@ -379,7 +388,7 @@ async fn handle_edu_direct_to_device( for (target_device_id_maybe, event) in map { match target_device_id_maybe { DeviceIdOrAllDevices::DeviceId(target_device_id) => { - services().users.add_to_device_event( + services.users.add_to_device_event( &sender, target_user_id, target_device_id, @@ -391,8 +400,8 @@ async fn handle_edu_direct_to_device( }, DeviceIdOrAllDevices::AllDevices => { - for target_device_id in services().users.all_device_ids(target_user_id) { - services().users.add_to_device_event( + for target_device_id in services.users.all_device_ids(target_user_id) { + services.users.add_to_device_event( &sender, target_user_id, &target_device_id?, @@ -408,7 +417,7 @@ async fn handle_edu_direct_to_device( } // Save transaction id with empty data - services() + services .transaction_ids .add_txnid(&sender, None, &message_id, &[])?; @@ -416,7 +425,7 @@ async fn handle_edu_direct_to_device( } async fn handle_edu_signing_key_update( - _client: &IpAddr, origin: &ServerName, content: SigningKeyUpdateContent, + services: &Services, _client: &IpAddr, origin: &ServerName, content: SigningKeyUpdateContent, ) -> Result<()> { let SigningKeyUpdateContent { user_id, @@ -433,7 +442,7 @@ async fn handle_edu_signing_key_update( } if let Some(master_key) = master_key { - services() + services .users .add_cross_signing_keys(&user_id, &master_key, &self_signing_key, &None, true)?; } From 923a98eb66aa74aaa1094d1efcf110ff6ee4246c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 15 Jul 2024 23:32:50 +0000 Subject: [PATCH 155/158] partially revert dc18f89c0b for now Signed-off-by: Jason Volk --- src/service/sending/resolve.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/service/sending/resolve.rs b/src/service/sending/resolve.rs index 8b6bfc95..e7a71af1 100644 --- a/src/service/sending/resolve.rs +++ b/src/service/sending/resolve.rs @@ -443,7 +443,9 @@ impl crate::globals::resolver::Resolver { impl CachedDest { #[inline] #[must_use] - pub fn valid(&self) -> bool { self.expire > SystemTime::now() } + pub fn valid(&self) -> bool { true } + + //pub fn valid(&self) -> bool { self.expire > SystemTime::now() } #[must_use] pub(crate) fn default_expire() -> SystemTime { rand::timepoint_secs(60 * 60 * 18..60 * 60 * 36) } @@ -452,7 +454,9 @@ impl CachedDest { impl CachedOverride { #[inline] #[must_use] - pub fn valid(&self) -> bool { self.expire > SystemTime::now() } + pub fn valid(&self) -> bool { true } + + //pub fn valid(&self) -> bool { self.expire > SystemTime::now() } #[must_use] pub(crate) fn default_expire() -> SystemTime { rand::timepoint_secs(60 * 60 * 6..60 * 60 * 12) } From 739eab46d534e897779ced2a189aa0384375302c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 15 Jul 2024 08:35:02 +0000 Subject: [PATCH 156/158] refactor thumbnail math Signed-off-by: Jason Volk --- src/service/media/thumbnail.rs | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/service/media/thumbnail.rs b/src/service/media/thumbnail.rs index 50be4ee1..01bf73f6 100644 --- a/src/service/media/thumbnail.rs +++ b/src/service/media/thumbnail.rs @@ -1,4 +1,4 @@ -use std::{io::Cursor, num::Saturating as Sat}; +use std::{cmp, io::Cursor, num::Saturating as Sat}; use conduit::{checked, Result}; use image::{imageops::FilterType, DynamicImage}; @@ -127,26 +127,29 @@ fn thumbnail_generate(image: &DynamicImage, width: u32, height: u32, crop: bool) } fn thumbnail_dimension(image: &DynamicImage, width: u32, height: u32) -> Result<(u32, u32)> { - let original_width = image.width(); - let original_height = image.height(); + let image_width = image.width(); + let image_height = image.height(); - let ratio = Sat(original_width) * Sat(height); - let nratio = Sat(width) * Sat(original_height); - let use_width = nratio <= ratio; + let width = cmp::min(width, image_width); + let height = cmp::min(height, image_height); - let intermediate = if use_width { - Sat(original_height) * Sat(checked!(width / original_width)?) + let use_width = Sat(width) * Sat(image_height) < Sat(height) * Sat(image_width); + + let x = if use_width { + let dividend = (Sat(height) * Sat(image_width)).0; + checked!(dividend / image_height)? } else { - Sat(original_width) * Sat(checked!(height / original_height)?) + width }; - let dims = if use_width { - (width, intermediate.0) + let y = if !use_width { + let dividend = (Sat(width) * Sat(image_height)).0; + checked!(dividend / image_width)? } else { - (intermediate.0, height) + height }; - Ok(dims) + Ok((x, y)) } /// Returns width, height of the thumbnail and whether it should be cropped. From c29197b3f457cf72197ef5251f9815107b2526d7 Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 16 Jul 2024 00:44:47 -0400 Subject: [PATCH 157/158] ci: dont publish `-rc` tags as `latest` docker branches Signed-off-by: strawberry --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9def2cf9..ef1c2b2e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,7 +60,7 @@ jobs: uses: actions/checkout@v4 - name: Tag comparison check - if: startsWith(github.ref, 'refs/tags/v') + if: ${{ startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') }} run: | # Tag mismatch with latest repo tag check to prevent potential downgrades LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1`) @@ -296,15 +296,15 @@ jobs: DOCKER_ARM64: docker.io/${{ github.repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-arm64v8 DOCKER_AMD64: docker.io/${{ github.repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-amd64 DOCKER_TAG: docker.io/${{ github.repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }} - DOCKER_BRANCH: docker.io/${{ github.repository }}:${{ (startsWith(github.ref, 'refs/tags/v') && 'latest') || (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }} + DOCKER_BRANCH: docker.io/${{ github.repository }}:${{ (startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') && 'latest') || (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }} GHCR_ARM64: ghcr.io/${{ github.repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-arm64v8 GHCR_AMD64: ghcr.io/${{ github.repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-amd64 GHCR_TAG: ghcr.io/${{ github.repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }} - GHCR_BRANCH: ghcr.io/${{ github.repository }}:${{ (startsWith(github.ref, 'refs/tags/v') && 'latest') || (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }} + GHCR_BRANCH: ghcr.io/${{ github.repository }}:${{ (startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') && 'latest') || (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }} GLCR_ARM64: registry.gitlab.com/conduwuit/conduwuit:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-arm64v8 GLCR_AMD64: registry.gitlab.com/conduwuit/conduwuit:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-amd64 GLCR_TAG: registry.gitlab.com/conduwuit/conduwuit:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }} - GLCR_BRANCH: registry.gitlab.com/conduwuit/conduwuit:${{ (startsWith(github.ref, 'refs/tags/v') && 'latest') || (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }} + GLCR_BRANCH: registry.gitlab.com/conduwuit/conduwuit:${{ (startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') && 'latest') || (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} From 5182a7e38fda780e294978cef8a2c0bcdbe6220b Mon Sep 17 00:00:00 2001 From: Matthew Scheffel Date: Tue, 16 Jul 2024 21:19:03 -0300 Subject: [PATCH 158/158] Set age dynamically #504 --- src/core/pdu/mod.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/core/pdu/mod.rs b/src/core/pdu/mod.rs index 0c54812e..a4254b08 100644 --- a/src/core/pdu/mod.rs +++ b/src/core/pdu/mod.rs @@ -116,7 +116,11 @@ impl PduEvent { .map_or_else(|| Ok(BTreeMap::new()), |u| serde_json::from_str(u.get())) .map_err(|_| Error::bad_database("Invalid unsigned in pdu event"))?; - unsigned.insert("age".to_owned(), to_raw_value(&1).unwrap()); + let now: u64 = MilliSecondsSinceUnixEpoch::now().get().into(); + let then: u64 = self.origin_server_ts.into(); + let this_age: u64 = now - then; + + unsigned.insert("age".to_owned(), to_raw_value(&this_age).unwrap()); self.unsigned = Some(to_raw_value(&unsigned).expect("unsigned is valid")); Ok(())