From d737e33b5ac4a09ada10de70b78b866481d69cba Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 29 Jun 2022 15:52:09 +0200 Subject: Rename MailIdent to UniqueIdent and use those to identify mailboxes --- src/mail/mail_ident.rs | 95 ------------------------------------------------ src/mail/mailbox.rs | 46 +++++++++-------------- src/mail/mod.rs | 4 +- src/mail/uidindex.rs | 42 ++++++++++----------- src/mail/unique_ident.rs | 95 ++++++++++++++++++++++++++++++++++++++++++++++++ src/mail/user.rs | 21 ++++++++--- 6 files changed, 150 insertions(+), 153 deletions(-) delete mode 100644 src/mail/mail_ident.rs create mode 100644 src/mail/unique_ident.rs (limited to 'src/mail') diff --git a/src/mail/mail_ident.rs b/src/mail/mail_ident.rs deleted file mode 100644 index 07e053a..0000000 --- a/src/mail/mail_ident.rs +++ /dev/null @@ -1,95 +0,0 @@ -use std::str::FromStr; -use std::sync::atomic::{AtomicU64, Ordering}; - -use lazy_static::lazy_static; -use rand::prelude::*; -use serde::{de::Error, Deserialize, Deserializer, Serialize, Serializer}; - -use crate::time::now_msec; - -/// An internal Mail Identifier is composed of two components: -/// - a process identifier, 128 bits, itself composed of: -/// - the timestamp of when the process started, 64 bits -/// - a 64-bit random number -/// - a sequence number, 64 bits -/// They are not part of the protocol but an internal representation -/// required by Mailrage/Aerogramme. -/// Their main property is to be unique without having to rely -/// on synchronization between IMAP processes. -#[derive(Clone, Copy, PartialOrd, Ord, PartialEq, Eq, Hash, Debug)] -pub struct MailIdent(pub [u8; 24]); - -struct IdentGenerator { - pid: u128, - sn: AtomicU64, -} - -impl IdentGenerator { - fn new() -> Self { - let time = now_msec() as u128; - let rand = thread_rng().gen::() as u128; - Self { - pid: (time << 64) | rand, - sn: AtomicU64::new(0), - } - } - - fn gen(&self) -> MailIdent { - let sn = self.sn.fetch_add(1, Ordering::Relaxed); - let mut res = [0u8; 24]; - res[0..16].copy_from_slice(&u128::to_be_bytes(self.pid)); - res[16..24].copy_from_slice(&u64::to_be_bytes(sn)); - MailIdent(res) - } -} - -lazy_static! { - static ref GENERATOR: IdentGenerator = IdentGenerator::new(); -} - -pub fn gen_ident() -> MailIdent { - GENERATOR.gen() -} - -// -- serde -- - -impl<'de> Deserialize<'de> for MailIdent { - fn deserialize(d: D) -> Result - where - D: Deserializer<'de>, - { - let v = String::deserialize(d)?; - MailIdent::from_str(&v).map_err(D::Error::custom) - } -} - -impl Serialize for MailIdent { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_str(&self.to_string()) - } -} - -impl ToString for MailIdent { - fn to_string(&self) -> String { - hex::encode(self.0) - } -} - -impl FromStr for MailIdent { - type Err = &'static str; - - fn from_str(s: &str) -> Result { - let bytes = hex::decode(s).map_err(|_| "invalid hex")?; - - if bytes.len() != 24 { - return Err("bad length"); - } - - let mut tmp = [0u8; 24]; - tmp[..].copy_from_slice(&bytes); - Ok(MailIdent(tmp)) - } -} diff --git a/src/mail/mailbox.rs b/src/mail/mailbox.rs index 9e8f0db..2e890d9 100644 --- a/src/mail/mailbox.rs +++ b/src/mail/mailbox.rs @@ -8,53 +8,40 @@ use tokio::sync::RwLock; use crate::bayou::Bayou; use crate::cryptoblob::Key; use crate::login::Credentials; -use crate::mail::mail_ident::*; use crate::mail::uidindex::*; +use crate::mail::unique_ident::*; use crate::mail::IMF; -pub struct Summary { - pub validity: ImapUidvalidity, - pub next: ImapUid, - pub exists: u32, - pub recent: u32, - pub flags: Vec, - pub unseen: Option, +pub struct Mailbox { + id: UniqueIdent, + mbox: RwLock, } -impl std::fmt::Display for Summary { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!( - f, - "uidvalidity: {}, uidnext: {}, exists: {}", - self.validity, self.next, self.exists - ) - } -} - -pub struct Mailbox(RwLock); - impl Mailbox { - pub(super) async fn open(creds: &Credentials, name: &str) -> Result { - let index_path = format!("index/{}", name); - let mail_path = format!("mail/{}", name); + pub(super) async fn open(creds: &Credentials, id: UniqueIdent) -> Result { + let index_path = format!("index/{}", id); + let mail_path = format!("mail/{}", id); let mut uid_index = Bayou::::new(creds, index_path)?; uid_index.sync().await?; - Ok(Self(RwLock::new(MailboxInternal { + let mbox = RwLock::new(MailboxInternal { + id, bucket: creds.bucket().to_string(), - key: creds.keys.master.clone(), + encryption_key: creds.keys.master.clone(), k2v: creds.k2v_client()?, s3: creds.s3_client()?, uid_index, mail_path, - }))) + }); + + Ok(Self { id, mbox }) } /// Get a clone of the current UID Index of this mailbox /// (cloning is cheap so don't hesitate to use this) pub async fn current_uid_index(&self) -> UidIndex { - self.0.read().await.uid_index.state().clone() + self.mbox.read().await.uid_index.state().clone() } /// Insert an email in the mailbox @@ -91,14 +78,15 @@ impl Mailbox { // Non standard but common flags: // https://www.iana.org/assignments/imap-jmap-keywords/imap-jmap-keywords.xhtml struct MailboxInternal { + id: UniqueIdent, bucket: String, - key: Key, + mail_path: String, + encryption_key: Key, k2v: K2vClient, s3: S3Client, uid_index: Bayou, - mail_path: String, } impl MailboxInternal { diff --git a/src/mail/mod.rs b/src/mail/mod.rs index 70182a9..7bee703 100644 --- a/src/mail/mod.rs +++ b/src/mail/mod.rs @@ -1,6 +1,6 @@ -pub mod mail_ident; pub mod mailbox; pub mod uidindex; +pub mod unique_ident; pub mod user; use std::convert::TryFrom; @@ -12,8 +12,8 @@ use rusoto_s3::S3Client; use crate::bayou::Bayou; use crate::cryptoblob::Key; use crate::login::Credentials; -use crate::mail::mail_ident::*; use crate::mail::uidindex::*; +use crate::mail::unique_ident::*; // Internet Message Format // aka RFC 822 - RFC 2822 - RFC 5322 diff --git a/src/mail/uidindex.rs b/src/mail/uidindex.rs index 9ffa386..7160ec6 100644 --- a/src/mail/uidindex.rs +++ b/src/mail/uidindex.rs @@ -4,7 +4,7 @@ use im::{HashMap, OrdMap, OrdSet}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use crate::bayou::*; -use crate::mail::mail_ident::MailIdent; +use crate::mail::unique_ident::UniqueIdent; pub type ImapUid = NonZeroU32; pub type ImapUidvalidity = NonZeroU32; @@ -18,10 +18,10 @@ pub type Flag = String; #[derive(Clone)] pub struct UidIndex { // Source of trust - pub table: OrdMap)>, + pub table: OrdMap)>, // Indexes optimized for queries - pub idx_by_uid: OrdMap, + pub idx_by_uid: OrdMap, pub idx_by_flag: FlagIndex, // Counters @@ -32,36 +32,36 @@ pub struct UidIndex { #[derive(Clone, Serialize, Deserialize, Debug)] pub enum UidIndexOp { - MailAdd(MailIdent, ImapUid, Vec), - MailDel(MailIdent), - FlagAdd(MailIdent, Vec), - FlagDel(MailIdent, Vec), + MailAdd(UniqueIdent, ImapUid, Vec), + MailDel(UniqueIdent), + FlagAdd(UniqueIdent, Vec), + FlagDel(UniqueIdent, Vec), } impl UidIndex { #[must_use] - pub fn op_mail_add(&self, ident: MailIdent, flags: Vec) -> UidIndexOp { + pub fn op_mail_add(&self, ident: UniqueIdent, flags: Vec) -> UidIndexOp { UidIndexOp::MailAdd(ident, self.internalseq, flags) } #[must_use] - pub fn op_mail_del(&self, ident: MailIdent) -> UidIndexOp { + pub fn op_mail_del(&self, ident: UniqueIdent) -> UidIndexOp { UidIndexOp::MailDel(ident) } #[must_use] - pub fn op_flag_add(&self, ident: MailIdent, flags: Vec) -> UidIndexOp { + pub fn op_flag_add(&self, ident: UniqueIdent, flags: Vec) -> UidIndexOp { UidIndexOp::FlagAdd(ident, flags) } #[must_use] - pub fn op_flag_del(&self, ident: MailIdent, flags: Vec) -> UidIndexOp { + pub fn op_flag_del(&self, ident: UniqueIdent, flags: Vec) -> UidIndexOp { UidIndexOp::FlagDel(ident, flags) } // INTERNAL functions to keep state consistent - fn reg_email(&mut self, ident: MailIdent, uid: ImapUid, flags: &Vec) { + fn reg_email(&mut self, ident: UniqueIdent, uid: ImapUid, flags: &Vec) { // Insert the email in our table self.table.insert(ident, (uid, flags.clone())); @@ -70,7 +70,7 @@ impl UidIndex { self.idx_by_flag.insert(uid, flags); } - fn unreg_email(&mut self, ident: &MailIdent) { + fn unreg_email(&mut self, ident: &UniqueIdent) { // We do nothing if the mail does not exist let (uid, flags) = match self.table.get(ident) { Some(v) => v, @@ -198,7 +198,7 @@ impl FlagIndex { #[derive(Serialize, Deserialize)] struct UidIndexSerializedRepr { - mails: Vec<(ImapUid, MailIdent, Vec)>, + mails: Vec<(ImapUid, UniqueIdent, Vec)>, uidvalidity: ImapUidvalidity, uidnext: ImapUid, internalseq: ImapUid, @@ -261,7 +261,7 @@ mod tests { // Add message 1 { - let m = MailIdent([0x01; 24]); + let m = UniqueIdent([0x01; 24]); let f = vec!["\\Recent".to_string(), "\\Archive".to_string()]; let ev = state.op_mail_add(m, f); state = state.apply(&ev); @@ -282,7 +282,7 @@ mod tests { // Add message 2 { - let m = MailIdent([0x02; 24]); + let m = UniqueIdent([0x02; 24]); let f = vec!["\\Seen".to_string(), "\\Archive".to_string()]; let ev = state.op_mail_add(m, f); state = state.apply(&ev); @@ -293,7 +293,7 @@ mod tests { // Add flags to message 1 { - let m = MailIdent([0x01; 24]); + let m = UniqueIdent([0x01; 24]); let f = vec!["Important".to_string(), "$cl_1".to_string()]; let ev = state.op_flag_add(m, f); state = state.apply(&ev); @@ -301,7 +301,7 @@ mod tests { // Delete flags from message 1 { - let m = MailIdent([0x01; 24]); + let m = UniqueIdent([0x01; 24]); let f = vec!["\\Recent".to_string()]; let ev = state.op_flag_del(m, f); state = state.apply(&ev); @@ -312,7 +312,7 @@ mod tests { // Delete message 2 { - let m = MailIdent([0x02; 24]); + let m = UniqueIdent([0x02; 24]); let ev = state.op_mail_del(m); state = state.apply(&ev); @@ -322,7 +322,7 @@ mod tests { // Add a message 3 concurrent to message 1 (trigger a uid validity change) { - let m = MailIdent([0x03; 24]); + let m = UniqueIdent([0x03; 24]); let f = vec!["\\Archive".to_string(), "\\Recent".to_string()]; let ev = UidIndexOp::MailAdd(m, 1, f); state = state.apply(&ev); @@ -334,7 +334,7 @@ mod tests { assert!(state.uidvalidity > 1); let (last_uid, ident) = state.idx_by_uid.get_max().unwrap(); - assert_eq!(ident, &MailIdent([0x03; 24])); + assert_eq!(ident, &UniqueIdent([0x03; 24])); let archive = state.idx_by_flag.0.get("\\Archive").unwrap(); assert_eq!(archive.len(), 2); diff --git a/src/mail/unique_ident.rs b/src/mail/unique_ident.rs new file mode 100644 index 0000000..6046281 --- /dev/null +++ b/src/mail/unique_ident.rs @@ -0,0 +1,95 @@ +use std::str::FromStr; +use std::sync::atomic::{AtomicU64, Ordering}; + +use lazy_static::lazy_static; +use rand::prelude::*; +use serde::{de::Error, Deserialize, Deserializer, Serialize, Serializer}; + +use crate::time::now_msec; + +/// An internal Mail Identifier is composed of two components: +/// - a process identifier, 128 bits, itself composed of: +/// - the timestamp of when the process started, 64 bits +/// - a 64-bit random number +/// - a sequence number, 64 bits +/// They are not part of the protocol but an internal representation +/// required by Mailrage/Aerogramme. +/// Their main property is to be unique without having to rely +/// on synchronization between IMAP processes. +#[derive(Clone, Copy, PartialOrd, Ord, PartialEq, Eq, Hash, Debug)] +pub struct UniqueIdent(pub [u8; 24]); + +struct IdentGenerator { + pid: u128, + sn: AtomicU64, +} + +impl IdentGenerator { + fn new() -> Self { + let time = now_msec() as u128; + let rand = thread_rng().gen::() as u128; + Self { + pid: (time << 64) | rand, + sn: AtomicU64::new(0), + } + } + + fn gen(&self) -> UniqueIdent { + let sn = self.sn.fetch_add(1, Ordering::Relaxed); + let mut res = [0u8; 24]; + res[0..16].copy_from_slice(&u128::to_be_bytes(self.pid)); + res[16..24].copy_from_slice(&u64::to_be_bytes(sn)); + UniqueIdent(res) + } +} + +lazy_static! { + static ref GENERATOR: IdentGenerator = IdentGenerator::new(); +} + +pub fn gen_ident() -> UniqueIdent { + GENERATOR.gen() +} + +// -- serde -- + +impl<'de> Deserialize<'de> for UniqueIdent { + fn deserialize(d: D) -> Result + where + D: Deserializer<'de>, + { + let v = String::deserialize(d)?; + UniqueIdent::from_str(&v).map_err(D::Error::custom) + } +} + +impl Serialize for UniqueIdent { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(&self.to_string()) + } +} + +impl std::fmt::Display for UniqueIdent { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", hex::encode(self.0)) + } +} + +impl FromStr for UniqueIdent { + type Err = &'static str; + + fn from_str(s: &str) -> Result { + let bytes = hex::decode(s).map_err(|_| "invalid hex")?; + + if bytes.len() != 24 { + return Err("bad length"); + } + + let mut tmp = [0u8; 24]; + tmp[..].copy_from_slice(&bytes); + Ok(UniqueIdent(tmp)) + } +} diff --git a/src/mail/user.rs b/src/mail/user.rs index e2b33e2..7b51623 100644 --- a/src/mail/user.rs +++ b/src/mail/user.rs @@ -9,6 +9,7 @@ use rusoto_s3::S3Client; use crate::login::{Credentials, StorageCredentials}; use crate::mail::mailbox::Mailbox; +use crate::mail::unique_ident::UniqueIdent; pub struct User { pub username: String, @@ -36,21 +37,29 @@ impl User { /// Opens an existing mailbox given its IMAP name. pub async fn open_mailbox(&self, name: &str) -> Result>> { + // TODO: handle mailbox names, mappings, renaming, etc + let id = match name { + "INBOX" => UniqueIdent([0u8; 24]), + _ => panic!("Only INBOX exists for now"), + }; + + let cache_key = (self.creds.storage.clone(), id); + { let cache = MAILBOX_CACHE.cache.lock().unwrap(); - if let Some(mb) = cache.get(&self.creds.storage).and_then(Weak::upgrade) { + if let Some(mb) = cache.get(&cache_key).and_then(Weak::upgrade) { return Ok(Some(mb)); } } - let mb = Arc::new(Mailbox::open(&self.creds, name).await?); + let mb = Arc::new(Mailbox::open(&self.creds, id).await?); let mut cache = MAILBOX_CACHE.cache.lock().unwrap(); - if let Some(concurrent_mb) = cache.get(&self.creds.storage).and_then(Weak::upgrade) { - drop(mb); // we worked for nothing but at least we didn't starve someone else + if let Some(concurrent_mb) = cache.get(&cache_key).and_then(Weak::upgrade) { + drop(mb); // we worked for nothing but at least we didn't starve someone else Ok(Some(concurrent_mb)) } else { - cache.insert(self.creds.storage.clone(), Arc::downgrade(&mb)); + cache.insert(cache_key, Arc::downgrade(&mb)); Ok(Some(mb)) } } @@ -74,7 +83,7 @@ impl User { // ---- Mailbox cache ---- struct MailboxCache { - cache: std::sync::Mutex>>, + cache: std::sync::Mutex>>, } impl MailboxCache { -- cgit v1.2.3