From b32bb6071a58c94dd014c53de989bff3e4cc166b Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Wed, 16 Aug 2023 18:01:22 +0200 Subject: [PATCH 1/6] partial re-implementation of body ext header fields is still missing --- Cargo.lock | 5 +- Cargo.toml | 2 +- src/imap/mailbox_view.rs | 208 ++++++++++++++++++++------------------- 3 files changed, 111 insertions(+), 104 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fa057ca..8a67046 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -843,9 +843,8 @@ checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" [[package]] name = "eml-codec" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac20cff537caf72385ffa5d9353ae63cb6c283a53665569408f040b8db36c90d" +version = "0.1.2" +source = "git+https://git.deuxfleurs.fr/Deuxfleurs/eml-codec.git?branch=main#5cff5510acc2c414b74419e0ee636d5ab249036b" dependencies = [ "base64 0.21.2", "chrono", diff --git a/Cargo.toml b/Cargo.toml index 6affde5..7d61962 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ backtrace = "0.3" base64 = "0.13" clap = { version = "3.1.18", features = ["derive", "env"] } duplexify = "1.1.0" -eml-codec = "0.1.1" +eml-codec = { git = "https://git.deuxfleurs.fr/Deuxfleurs/eml-codec.git", branch = "main" } hex = "0.4" futures = "0.3" im = "15" diff --git a/src/imap/mailbox_view.rs b/src/imap/mailbox_view.rs index 2124855..72d39af 100644 --- a/src/imap/mailbox_view.rs +++ b/src/imap/mailbox_view.rs @@ -1,3 +1,4 @@ +use std::borrow::Cow; use std::num::NonZeroU32; use std::sync::Arc; @@ -17,8 +18,8 @@ use imap_codec::types::flag::{Flag, StoreResponse, StoreType}; use imap_codec::types::response::{Code, Data, MessageAttribute, Status}; use imap_codec::types::sequence::{self, SequenceSet}; use eml_codec::{ - imf::{self as imf}, - part::{AnyPart}, + imf, + part::{AnyPart, composite::Message}, mime::r#type::Deductible, mime, }; @@ -37,6 +38,26 @@ const DEFAULT_FLAGS: [Flag; 5] = [ const BODY_CHECK: &str = "body attribute asked but only header is fetched, logic error"; +enum FetchedMail<'a> { + Partial(imf::Imf<'a>), + Full(Message<'a>), +} +impl<'a> FetchedMail<'a> { + fn as_full(&self) -> Result<&Message<'a>> { + match self { + FetchedMail::Full(x) => Ok(&x), + _ => bail!("The full message must be fetched, not only its headers: it's a logic error"), + } + } + + fn imf(&self) -> &imf::Imf<'a> { + match self { + FetchedMail::Full(x) => &x.imf, + FetchedMail::Partial(x) => &x, + } + } +} + /// A MailboxView is responsible for giving the client the information /// it needs about a mailbox, such as an initial summary of the mailbox's /// content and continuous updates indicating when the content @@ -288,14 +309,12 @@ impl MailboxView { .get(&uuid) .ok_or_else(|| anyhow!("Mail not in uidindex table: {}", uuid))?; - let (parts, imf) = match &body { + let fetched = match &body { Some(m) => { - let eml = eml_codec::parse_message(m).or(Err(anyhow!("Invalid mail body")))?.1; - (Some(eml.child), eml.imf) + FetchedMail::Full(eml_codec::parse_message(m).or(Err(anyhow!("Invalid mail body")))?.1) } None => { - let imf = eml_codec::parse_imf(&meta.headers).or(Err(anyhow!("Invalid mail headers")))?.1; - (None, imf) + FetchedMail::Partial(eml_codec::parse_imf(&meta.headers).or(Err(anyhow!("Invalid mail headers")))?.1) } }; @@ -316,14 +335,8 @@ impl MailboxView { ))) } FetchAttribute::Rfc822Text => { - //@FIXME this is not efficient, this is a hack as we need to patch - // eml_codec to correctly implement this behavior - let txt = eml_codec::parse_imf(body.as_ref().expect(BODY_CHECK).as_slice()) - .map(|(x, _)| x) - .unwrap_or(b""); - attributes.push(MessageAttribute::Rfc822Text(NString( - txt.try_into().ok().map(IString::Literal), + fetched.as_full()?.raw_body.try_into().ok().map(IString::Literal), ))); } FetchAttribute::Rfc822 => attributes.push(MessageAttribute::Rfc822(NString( @@ -335,23 +348,29 @@ impl MailboxView { .map(IString::Literal), ))), FetchAttribute::Envelope => { - attributes.push(MessageAttribute::Envelope(message_envelope(&imf))) + attributes.push(MessageAttribute::Envelope(message_envelope(fetched.imf()))) } FetchAttribute::Body => attributes.push(MessageAttribute::Body( - build_imap_email_struct(parts.as_ref().expect(BODY_CHECK).as_ref())?, + build_imap_email_struct(fetched.as_full()?.child.as_ref())?, )), FetchAttribute::BodyStructure => attributes.push(MessageAttribute::Body( - build_imap_email_struct(parts.as_ref().expect(BODY_CHECK).as_ref())?, + build_imap_email_struct(fetched.as_full()?.child.as_ref())?, )), + + // maps to BODY[
]<> and BODY.PEEK[
]<> + // peek does not implicitly set the \Seen flag + // + // eg. BODY[HEADER.FIELDS (DATE FROM)] + // eg. BODY[]<0.2048> FetchAttribute::BodyExt { section, partial, peek, } => { - // @FIXME deactivated while eml_codec is integrated - todo!(); // @TODO Add missing section specifiers - /*match get_message_section(&parts.expect("body attribute asked but only header is fetched, logic error"), section) { + + // Extract message section + match get_message_section(fetched.as_full()?, section) { Ok(text) => { let seen_flag = Flag::Seen.to_string(); if !peek && !flags.iter().any(|x| *x == seen_flag) { @@ -359,6 +378,7 @@ impl MailboxView { self.mailbox.add_flags(uuid, &[seen_flag]).await?; } + // Handle <> which cut the message byts let (text, origin) = match partial { Some((begin, len)) => { if *begin as usize > text.len() { @@ -393,7 +413,6 @@ impl MailboxView { ); } } - */ } FetchAttribute::InternalDate => { let dt = Utc.fix().timestamp_opt(i64::try_from(meta.internaldate / 1000)?, 0).earliest().ok_or(anyhow!("Unable to parse internal date"))?; @@ -817,54 +836,61 @@ fn basic_fields(m: &mime::NaiveMIME, sz: usize) -> Result { }) } -/* +/// Extract message section for section identifier passed by the FETCH BODY[
]<> +/// request +/// +/// Example of message sections: +/// +/// ``` +/// HEADER ([RFC-2822] header of the message) +/// TEXT ([RFC-2822] text body of the message) MULTIPART/MIXED +/// 1 TEXT/PLAIN +/// 2 APPLICATION/OCTET-STREAM +/// 3 MESSAGE/RFC822 +/// 3.HEADER ([RFC-2822] header of the message) +/// 3.TEXT ([RFC-2822] text body of the message) MULTIPART/MIXED +/// 3.1 TEXT/PLAIN +/// 3.2 APPLICATION/OCTET-STREAM +/// 4 MULTIPART/MIXED +/// 4.1 IMAGE/GIF +/// 4.1.MIME ([MIME-IMB] header for the IMAGE/GIF) +/// 4.2 MESSAGE/RFC822 +/// 4.2.HEADER ([RFC-2822] header of the message) +/// 4.2.TEXT ([RFC-2822] text body of the message) MULTIPART/MIXED +/// 4.2.1 TEXT/PLAIN +/// 4.2.2 MULTIPART/ALTERNATIVE +/// 4.2.2.1 TEXT/PLAIN +/// 4.2.2.2 TEXT/RICHTEXT +/// ``` fn get_message_section<'a>( parsed: &'a Message<'a>, section: &Option, ) -> Result> { match section { Some(FetchSection::Text(None)) => { - let rp = parsed.root_part(); - Ok(parsed - .raw_message - .get(rp.offset_body..rp.offset_end) - .ok_or(Error::msg( - "Unable to extract email body, cursors out of bound. This is a bug.", - ))? - .into()) + Ok(parsed.raw_body.into()) } Some(FetchSection::Text(Some(part))) => { - map_subpart_msg(parsed, part.0.as_slice(), |part_msg| { - let rp = part_msg.root_part(); - Ok(part_msg - .raw_message - .get(rp.offset_body..rp.offset_end) - .ok_or(Error::msg( - "Unable to extract email body, cursors out of bound. This is a bug.", - ))? - .to_vec() + map_subpart(parsed.child.as_ref(), part.0.as_slice(), |part_msg| { + Ok(part_msg.as_message().ok_or(Error::msg("Not a message/rfc822 part while expected by request (xxx.TEXT)"))? + .raw_body .into()) }) } - Some(FetchSection::Header(part)) => map_subpart_msg( - parsed, + Some(FetchSection::Header(part)) => map_subpart( + parsed.child.as_ref(), part.as_ref().map(|p| p.0.as_slice()).unwrap_or(&[]), |part_msg| { - let rp = part_msg.root_part(); - Ok(part_msg - .raw_message - .get(..rp.offset_body) - .ok_or(Error::msg( - "Unable to extract email header, cursors out of bound. This is a bug.", - ))? - .to_vec() + Ok(part_msg.as_message().ok_or(Error::msg("Not a message/rfc822 part while expected by request (xxx.TEXT)"))? + .raw_headers .into()) }, ), Some( FetchSection::HeaderFields(part, fields) | FetchSection::HeaderFieldsNot(part, fields), ) => { - let invert = matches!(section, Some(FetchSection::HeaderFieldsNot(_, _))); + todo!(); + /*let invert = matches!(section, Some(FetchSection::HeaderFieldsNot(_, _))); let fields = fields .iter() .map(|x| match x { @@ -893,71 +919,53 @@ fn get_message_section<'a>( ret.extend(b"\r\n"); Ok(ret.into()) }, - ) + )*/ } - Some(FetchSection::Part(part)) => map_subpart(parsed, part.0.as_slice(), |_msg, part| { - let bytes = match &part.body { - AnyPart::Txt(p) => p.as_bytes().to_vec(), - AnyPart::Bin(p) => p.to_vec(), - AnyPart::Msg(p) => p.raw_message.to_vec(), - AnyPart::Multipart(_) => bail!("Multipart part has no body"), + Some(FetchSection::Part(part)) => map_subpart(parsed.child.as_ref(), part.0.as_slice(), |part| { + let bytes = match &part { + AnyPart::Txt(p) => p.body, + AnyPart::Bin(p) => p.body, + AnyPart::Msg(p) => p.raw_part, + AnyPart::Mult(_) => bail!("Multipart part has no body"), }; - Ok(bytes.into()) + Ok(bytes.to_vec().into()) }), - Some(FetchSection::Mime(part)) => map_subpart(parsed, part.0.as_slice(), |msg, part| { - let mut ret = vec![]; - for head in part.headers.iter() { - ret.extend(head.name.as_str().as_bytes()); - ret.extend(b": "); - ret.extend(&msg.raw_message[head.offset_start..head.offset_end]); - } - ret.extend(b"\r\n"); - Ok(ret.into()) + Some(FetchSection::Mime(part)) => map_subpart(parsed.child.as_ref(), part.0.as_slice(), |part| { + let bytes = match &part { + AnyPart::Txt(p) => p.mime.fields.raw, + AnyPart::Bin(p) => p.mime.fields.raw, + AnyPart::Msg(p) => p.mime.fields.raw, + AnyPart::Mult(p) => p.mime.fields.raw, + }; + Ok(bytes.to_vec().into()) }), - None => Ok(parsed.raw_message.clone()), + None => Ok(parsed.raw_part.into()), } } -fn map_subpart_msg(msg: &Message<'_>, path: &[NonZeroU32], f: F) -> Result +/// Fetch a MIME SubPart +/// +/// eg. FETCH BODY[4.2.2.1] -> [4, 2, 2, 1] +fn map_subpart<'a, F, R>(part: &AnyPart<'a>, path: &[NonZeroU32], f: F) -> Result where - F: FnOnce(&Message<'_>) -> Result, + F: FnOnce(&AnyPart<'a>) -> Result, { if path.is_empty() { - f(msg) + f(part) } else { - let part = msg - .parts - .get(path[0].get() as usize - 1) - .ok_or(anyhow!("No such subpart: {}", path[0]))?; - if let PartType::Message(msg_attach) = &part.body { - map_subpart_msg(msg_attach, &path[1..], f) - } else { - bail!("Subpart is not a message: {}", path[0]); + match part { + AnyPart::Mult(x) => map_subpart( + x.children + .get(path[0].get() as usize - 1) + .ok_or(anyhow!("Unable to resolve subpath {:?}, current multipart has only {} elements", path, x.children.len()))?, + &path[1..], + f), + AnyPart::Msg(x) => map_subpart(x.child.as_ref(), path, f), + _ => bail!("You tried to access a subpart on an atomic part (text or binary). Unresolved subpath {:?}", path), } } } -fn map_subpart(msg: &Message<'_>, path: &[NonZeroU32], f: F) -> Result -where - F: FnOnce(&Message<'_>, &MessagePart<'_>) -> Result, -{ - if path.is_empty() { - bail!("Unexpected empty path"); - } else { - let part = msg - .parts - .get(path[0].get() as usize - 1) - .ok_or(anyhow!("No such subpart: {}", path[0]))?; - if path.len() == 1 { - f(msg, part) - } else if let PartType::Message(msg_attach) = &part.body { - map_subpart(msg_attach, &path[1..], f) - } else { - bail!("Subpart is not a message: {}", path[0]); - } - } -}*/ - #[cfg(test)] mod tests { use super::*; From 1fb9970502ed654d99ab41dd8d1cb5c09ea91c02 Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Thu, 21 Sep 2023 11:27:33 +0200 Subject: [PATCH 2/6] add back header fields --- Cargo.lock | 2 +- src/imap/mailbox_view.rs | 24 +++++++++++++++--------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8a67046..a5fe09e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -844,7 +844,7 @@ checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" [[package]] name = "eml-codec" version = "0.1.2" -source = "git+https://git.deuxfleurs.fr/Deuxfleurs/eml-codec.git?branch=main#5cff5510acc2c414b74419e0ee636d5ab249036b" +source = "git+https://git.deuxfleurs.fr/Deuxfleurs/eml-codec.git?branch=main#a7bd3c475a58e42b86c163ec075ce01ddae7e60a" dependencies = [ "base64 0.21.2", "chrono", diff --git a/src/imap/mailbox_view.rs b/src/imap/mailbox_view.rs index 72d39af..d04fabf 100644 --- a/src/imap/mailbox_view.rs +++ b/src/imap/mailbox_view.rs @@ -18,6 +18,7 @@ use imap_codec::types::flag::{Flag, StoreResponse, StoreType}; use imap_codec::types::response::{Code, Data, MessageAttribute, Status}; use imap_codec::types::sequence::{self, SequenceSet}; use eml_codec::{ + header, imf, part::{AnyPart, composite::Message}, mime::r#type::Deductible, @@ -889,8 +890,7 @@ fn get_message_section<'a>( Some( FetchSection::HeaderFields(part, fields) | FetchSection::HeaderFieldsNot(part, fields), ) => { - todo!(); - /*let invert = matches!(section, Some(FetchSection::HeaderFieldsNot(_, _))); + let invert = matches!(section, Some(FetchSection::HeaderFieldsNot(_, _))); let fields = fields .iter() .map(|x| match x { @@ -900,26 +900,32 @@ fn get_message_section<'a>( }) .collect::>(); - map_subpart_msg( - parsed, + + map_subpart( + parsed.child.as_ref(), part.as_ref().map(|p| p.0.as_slice()).unwrap_or(&[]), |part_msg| { let mut ret = vec![]; - for (hn, hv) in part_msg.headers_raw() { + for f in &part_msg.mime().kv { + let (k, v) = match f { + header::Field::Good(header::Kv2(k, v)) => (k, v), + _ => continue, + }; if fields .as_slice() .iter() - .any(|x| (*x == hn.as_bytes()) ^ invert) + .any(|x| (x == k) ^ invert) { - ret.extend(hn.as_bytes()); + ret.extend(*k); ret.extend(b": "); - ret.extend(hv.as_bytes()); + ret.extend(*v); + ret.extend(b"\r\n"); } } ret.extend(b"\r\n"); Ok(ret.into()) }, - )*/ + ) } Some(FetchSection::Part(part)) => map_subpart(parsed.child.as_ref(), part.0.as_slice(), |part| { let bytes = match &part { From 2270aaa96369d155cd3dfacf54487f974e1ef72b Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Thu, 28 Sep 2023 11:57:46 +0200 Subject: [PATCH 3/6] WIP --- src/imap/mailbox_view.rs | 149 ++++++++++++++++++++++++++++++++------- 1 file changed, 123 insertions(+), 26 deletions(-) diff --git a/src/imap/mailbox_view.rs b/src/imap/mailbox_view.rs index d04fabf..1b9337e 100644 --- a/src/imap/mailbox_view.rs +++ b/src/imap/mailbox_view.rs @@ -59,6 +59,96 @@ impl<'a> FetchedMail<'a> { } } +pub struct MailIdentifiers { + i: NonZeroU32, + uid: ImapUid, + uuid: UniqueIdent, +} + +pub struct MailView<'a> { + ids: MailIdentifiers, + meta: MailMeta, + flags: Vec, + content: FetchedMail<'a>, +} + +pub struct AttributesProxy { + attrs: FetchAttributes +} +impl AttributeProxy { + fn new(attrs: &MacroOrFetchAttributes, is_uid_fetch: bool) -> Self { + // Expand macros + let mut fetch_attrs = match attributes { + MacroOrFetchAttributes::Macro(m) => m.expand(), + MacroOrFetchAttributes::FetchAttributes(a) => a.clone(), + }; + + // Handle uids + if *is_uid_fetch && !fetch_attrs.contains(&FetchAttribute::Uid) { + fetch_attrs.push(FetchAttribute::Uid); + } + + Self { attrs: fetch_attrs } + } + + fn need_body(&self) -> bool { + self.attrs.iter().any(|x| { + matches!( + x, + FetchAttribute::Body + | FetchAttribute::BodyExt { .. } + | FetchAttribute::Rfc822 + | FetchAttribute::Rfc822Text + | FetchAttribute::BodyStructure + ) + }) + } +} + +#[derive(Default)] +pub BatchMailViewBuilder<'a> struct { + attrs: AttributeProxy, + mi: Vec, + meta: Vec, + flags: Vec>, + bodies: Vec>, +} +impl BatchMailViewBuilder<'a> { + fn new(req_attr: &MacroOrFetchAttributes) -> Self { + Self { + attrs: AttributeProxy::new(req_attr) + } + } + + fn with_mail_identifiers(mut self, mi: Vec) -> Self { + self.mi = mi; + self + } + + fn with_metadata(mut self, meta: Vec) -> Self { + self.meta = meta; + self + } + + fn with_flags(mut self, flags: Vec>) -> Self { + self.flags = flags; + self + } + + fn collect_bodies(mut self, FnOnce) -> Self { + if self.attrs.need_body() { + + } else { + self.bodies = + } + self + } + + fn build(self) -> Result> { + + } +} + /// A MailboxView is responsible for giving the client the information /// it needs about a mailbox, such as an initial summary of the mailbox's /// content and continuous updates indicating when the content @@ -252,32 +342,29 @@ impl MailboxView { attributes: &MacroOrFetchAttributes, is_uid_fetch: &bool, ) -> Result> { + + /* + let mails = self.get_mail_ids(sequence_set, *is_uid_fetch)?; + let mails_uuid = mails + .iter() + .map(|mi| mi.uuid) + .collect::>(); + + let mails_meta = self.mailbox.fetch_meta(&mails_uuid).await?; + */ + + // mail identifiers let mails = self.get_mail_ids(sequence_set, *is_uid_fetch)?; let mails_uuid = mails .iter() .map(|(_i, _uid, uuid)| *uuid) .collect::>(); + + // mail metadata let mails_meta = self.mailbox.fetch_meta(&mails_uuid).await?; - let mut fetch_attrs = match attributes { - MacroOrFetchAttributes::Macro(m) => m.expand(), - MacroOrFetchAttributes::FetchAttributes(a) => a.clone(), - }; - if *is_uid_fetch && !fetch_attrs.contains(&FetchAttribute::Uid) { - fetch_attrs.push(FetchAttribute::Uid); - } - let need_body = fetch_attrs.iter().any(|x| { - matches!( - x, - FetchAttribute::Body - | FetchAttribute::BodyExt { .. } - | FetchAttribute::Rfc822 - | FetchAttribute::Rfc822Text - | FetchAttribute::BodyStructure - ) - }); - + // fectch email body and transform to a state usable by select let mails = if need_body { let mut iter = mails .into_iter() @@ -300,16 +387,16 @@ impl MailboxView { .collect::>() }; - let mut ret = vec![]; - for (i, uid, uuid, meta, body) in mails { - let mut attributes = vec![]; - - let (_uid2, flags) = self + // add flags + let mails = mails.into_iter().filter_map(|(i, uid, uuid, meta, body)| + self .known_state .table .get(&uuid) - .ok_or_else(|| anyhow!("Mail not in uidindex table: {}", uuid))?; + .map(|(_uid2, flags)| (i, uid, uuid, meta, flags, body))) + .collect::>(); + // parse email body / headers let fetched = match &body { Some(m) => { FetchedMail::Full(eml_codec::parse_message(m).or(Err(anyhow!("Invalid mail body")))?.1) @@ -319,6 +406,16 @@ impl MailboxView { } }; + // do the logic + select_mail_fragments(fetch_attrs, mails) + } + + todo!(); + + let mut ret = vec![]; + for (i, uid, uuid, meta, flags, body) in mails { + let mut attributes = vec![]; + for attr in fetch_attrs.iter() { match attr { FetchAttribute::Uid => attributes.push(MessageAttribute::Uid(uid)), @@ -433,8 +530,8 @@ impl MailboxView { // ---- - // Gets the UIDs and UUIDs of mails identified by a SequenceSet of - // sequence numbers + // Gets the IMAP ID, the IMAP UIDs and, the Aerogramme UUIDs of mails identified by a SequenceSet of + // sequence numbers (~ IMAP selector) fn get_mail_ids( &self, sequence_set: &SequenceSet, From f24c06312bdd2fec18abe8e8002aa116b1c4ade5 Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Mon, 9 Oct 2023 12:00:16 +0200 Subject: [PATCH 4/6] WIP refactor, support LSP --- flake.nix | 1 + src/imap/mailbox_view.rs | 462 +++++++++++++++++++++------------------ 2 files changed, 249 insertions(+), 214 deletions(-) diff --git a/flake.nix b/flake.nix index c1561f4..8304326 100644 --- a/flake.nix +++ b/flake.nix @@ -61,6 +61,7 @@ ]; shellHook = '' echo "AEROGRAME DEVELOPMENT SHELL ${fenix.packages.x86_64-linux.minimal.rustc}" + export RUST_SRC_PATH="${fenix.packages.x86_64-linux.latest.rust-src}/lib/rustlib/src/rust/library" ''; }; diff --git a/src/imap/mailbox_view.rs b/src/imap/mailbox_view.rs index 1b9337e..1378378 100644 --- a/src/imap/mailbox_view.rs +++ b/src/imap/mailbox_view.rs @@ -1,11 +1,15 @@ use std::borrow::Cow; use std::num::NonZeroU32; use std::sync::Arc; +use std::iter::zip; use anyhow::{anyhow, bail, Error, Result}; use boitalettres::proto::res::body::Data as Body; use chrono::{Offset, TimeZone, Utc}; + +use futures::future::join_all; use futures::stream::{FuturesOrdered, StreamExt}; + use imap_codec::types::address::Address; use imap_codec::types::body::{BasicFields, Body as FetchBody, BodyStructure, SpecificFields}; use imap_codec::types::core::{AString, Atom, IString, NString}; @@ -17,6 +21,7 @@ use imap_codec::types::fetch_attributes::{ use imap_codec::types::flag::{Flag, StoreResponse, StoreType}; use imap_codec::types::response::{Code, Data, MessageAttribute, Status}; use imap_codec::types::sequence::{self, SequenceSet}; + use eml_codec::{ header, imf, @@ -25,7 +30,8 @@ use eml_codec::{ mime, }; -use crate::mail::mailbox::Mailbox; +use crate::cryptoblob::Key; +use crate::mail::mailbox::{Mailbox, MailMeta}; use crate::mail::uidindex::{ImapUid, ImapUidvalidity, UidIndex}; use crate::mail::unique_ident::UniqueIdent; @@ -37,8 +43,6 @@ const DEFAULT_FLAGS: [Flag; 5] = [ Flag::Draft, ]; -const BODY_CHECK: &str = "body attribute asked but only header is fetched, logic error"; - enum FetchedMail<'a> { Partial(imf::Imf<'a>), Full(Message<'a>), @@ -59,26 +63,13 @@ impl<'a> FetchedMail<'a> { } } -pub struct MailIdentifiers { - i: NonZeroU32, - uid: ImapUid, - uuid: UniqueIdent, -} - -pub struct MailView<'a> { - ids: MailIdentifiers, - meta: MailMeta, - flags: Vec, - content: FetchedMail<'a>, -} - pub struct AttributesProxy { - attrs: FetchAttributes + attrs: Vec } -impl AttributeProxy { +impl AttributesProxy { fn new(attrs: &MacroOrFetchAttributes, is_uid_fetch: bool) -> Self { // Expand macros - let mut fetch_attrs = match attributes { + let mut fetch_attrs = match attrs { MacroOrFetchAttributes::Macro(m) => m.expand(), MacroOrFetchAttributes::FetchAttributes(a) => a.clone(), }; @@ -105,47 +96,230 @@ impl AttributeProxy { } } +pub struct MailIdentifiers { + i: NonZeroU32, + uid: ImapUid, + uuid: UniqueIdent, +} + +pub struct MailView<'a> { + ids: &'a MailIdentifiers, + meta: &'a MailMeta, + flags: &'a Vec, + content: FetchedMail<'a>, + add_seen: bool +} + +impl<'a> MailView<'a> { + fn uid(&self) -> MessageAttribute { + MessageAttribute::Uid(self.ids.uid) + } + + fn flags(&self) -> MessageAttribute { + MessageAttribute::Flags( + self.flags.iter().filter_map(|f| string_to_flag(f)).collect(), + ) + } + + fn rfc_822_size(&self) -> MessageAttribute { + MessageAttribute::Rfc822Size(self.meta.rfc822_size as u32) + } + + fn rfc_822_header(&self) -> MessageAttribute { + MessageAttribute::Rfc822Header(NString(self.meta.headers.to_vec().try_into().ok().map(IString::Literal))) + } + + fn rfc_822_text(&self) -> Result { + Ok(MessageAttribute::Rfc822Text(NString( + self.content.as_full()?.raw_body.try_into().ok().map(IString::Literal), + ))) + } + + fn rfc822(&self) -> Result { + Ok(MessageAttribute::Rfc822(NString( + self.content.as_full()?.raw_body.clone().try_into().ok().map(IString::Literal)))) + } + + fn envelope(&self) -> MessageAttribute { + message_envelope(self.content.imf()) + } + + fn body(&self) -> Result { + Ok(MessageAttribute::Body( + build_imap_email_struct(self.content.as_full()?.child.as_ref())?, + )) + } + + fn body_structure(&self) -> Result { + Ok(MessageAttribute::Body( + build_imap_email_struct(self.content.as_full()?.child.as_ref())?, + )) + } + + /// maps to BODY[
]<> and BODY.PEEK[
]<> + /// peek does not implicitly set the \Seen flag + /// eg. BODY[HEADER.FIELDS (DATE FROM)] + /// eg. BODY[]<0.2048> + fn body_ext(&mut self, section: Option, partial: Option<(u32, NonZeroU32)>, peek: bool) -> Result> { + // Extract message section + match get_message_section(self.content.as_full()?, section) { + Ok(text) => { + let seen_flag = Flag::Seen.to_string(); + if !peek && !self.flags.iter().any(|x| *x == seen_flag) { + // Add \Seen flag + //self.mailbox.add_flags(uuid, &[seen_flag]).await?; + self.add_seen = true; + } + + // Handle <> which cut the message bytes + let (text, origin) = apply_partial(partial, text); + + let data = NString(text.to_vec().try_into().ok().map(IString::Literal)); + + return Ok(Some(MessageAttribute::BodyExt { + section: section.clone(), + origin, + data, + })) + } + Err(e) => { + tracing::error!( + "Could not get section {:?} of message {}: {}", + section, + self.mi.uuid, + e + ); + return Ok(None) + } + } + } + + fn internal_date(&self) -> Result { + let dt = Utc.fix().timestamp_opt(i64::try_from(self.meta.internaldate / 1000)?, 0).earliest().ok_or(anyhow!("Unable to parse internal date"))?; + Ok(MessageAttribute::InternalDate(MyDateTime(dt))) + } + + fn filter(&mut self, ap: &AttributesProxy) -> Result> { + let res_attrs = ap.attrs.iter().filter_map(|attr| match attr { + FetchAttribute::Uid => Some(self.uid()), + FetchAttribute::Flags => Some(self.flags()), + FetchAttribute::Rfc822Size => Some(self.rfc_822_size()), + FetchAttribute::Rfc822Header => Some(self.rfc_822_header()), + FetchAttribute::Rfc822Text => Some(self.rfc_822_text()?), + FetchAttribute::Rfc822 => Some(self.rfc822()?), + FetchAttribute::Envelope => Some(self.envelope()), + FetchAttribute::Body => Some(self.body()?), + FetchAttribute::BodyStructure => Some(self.body_structure()?), + FetchAttribute::BodyExt { section, partial, peek } => self.body_ext(section, partial, peek)?, + FetchAttribute::InternalDate => Some(self.internal_date()?) + }).collect::>(); + + Body::Data(Data::Fetch { + seq_or_uid: self.ids.i, + res_attrs, + }) + } +} + +fn apply_partial(partial: Option<(u32, NonZeroU32)>, text: &[u8]) -> (&[u8], Option) { + match partial { + Some((begin, len)) => { + if *begin as usize > text.len() { + (&[][..], Some(*begin)) + } else if (*begin + len.get()) as usize >= text.len() { + (&text[*begin as usize..], Some(*begin)) + } else { + ( + &text[*begin as usize..(*begin + len.get()) as usize], + Some(*begin), + ) + } + } + None => (&text[..], None), + }; + +} + +pub struct BodyIdentifier<'a> { + msg_uuid: &'a UniqueIdent, + msg_key: &'a Key, +} + #[derive(Default)] -pub BatchMailViewBuilder<'a> struct { - attrs: AttributeProxy, - mi: Vec, - meta: Vec, - flags: Vec>, +pub struct MailSelectionBuilder<'a> { + //attrs: AttributeProxy, + mail_count: usize, + need_body: bool, + mi: &'a [MailIdentifiers], + meta: &'a [MailMeta], + flags: &'a [Vec], bodies: Vec>, } -impl BatchMailViewBuilder<'a> { - fn new(req_attr: &MacroOrFetchAttributes) -> Self { - Self { - attrs: AttributeProxy::new(req_attr) + +impl<'a> MailSelectionBuilder<'a> { + fn new(need_body: bool, mail_count: usize) -> Self { + Self { + mail_count, + need_body, + bodies: vec![0; mail_count], + ..MailSelectionBuilder::default() } } - fn with_mail_identifiers(mut self, mi: Vec) -> Self { + fn with_mail_identifiers(mut self, mi: &[MailIdentifiers]) -> Self { self.mi = mi; self } - fn with_metadata(mut self, meta: Vec) -> Self { + fn uuids(&self) -> Vec<&UniqueIdent> { + self.mi.iter().map(|i| i.uuid ).collect::>() + } + + fn with_metadata(mut self, meta: &[MailMeta]) -> Self { self.meta = meta; self } - fn with_flags(mut self, flags: Vec>) -> Self { + fn with_flags(mut self, flags: &[Vec]) -> Self { self.flags = flags; self } - fn collect_bodies(mut self, FnOnce) -> Self { - if self.attrs.need_body() { - - } else { - self.bodies = + fn bodies_to_collect(&self) -> Vec { + if !self.attrs.need_body() { + return vec![] } + zip(self.mi, self.meta).as_ref().iter().enumerate().map(|(i , (mi, k))| BodyIdentifier { i, mi, k }).collect::>() + } + + fn with_bodies(mut self, rbodies: &[&'a [u8]]) -> Self { + for rb in rbodies.iter() { + let (_, p) = eml_codec::parse_message(&rb).or(Err(anyhow!("Invalid mail body")))?; + self.bodies.push(FetchedMail::Full(p)); + } + self } - fn build(self) -> Result> { + fn build(self) -> Result>> { + if !self.need_body && self.bodies.len() == 0 { + for m in self.meta.iter() { + let (_, hdrs) = eml_codec::parse_imf(&self.meta.headers).or(Err(anyhow!("Invalid mail headers")))?; + self.bodies.push(FetchedMail::Partial(hdrs)); + } + } + if self.mi.len() != self.mail_count && self.meta.len() != self.mail_count || self.flags.len() != self.mail_count || self.bodies.len() != self.mail_count { + return Err(anyhow!("Can't build a mail view selection as parts were not correctly registered into the builder.")) + } + + let views = Vec::with_capacity(self.mail_count); + for (ids, meta, flags, content) in zip(self.mi, self.meta, self.flags, self.bodies) { + let mv = MailView { ids, meta, flags, content }; + views.push(mv); + } + + return Ok(views) } } @@ -343,188 +517,48 @@ impl MailboxView { is_uid_fetch: &bool, ) -> Result> { - /* - let mails = self.get_mail_ids(sequence_set, *is_uid_fetch)?; - let mails_uuid = mails - .iter() - .map(|mi| mi.uuid) - .collect::>(); - - let mails_meta = self.mailbox.fetch_meta(&mails_uuid).await?; - */ + let ap = AttributesProxy::new(attributes, *is_uid_fetch); + let mail_count = sequence_set.iter().count(); - // mail identifiers - let mails = self.get_mail_ids(sequence_set, *is_uid_fetch)?; - - let mails_uuid = mails - .iter() - .map(|(_i, _uid, uuid)| *uuid) - .collect::>(); - - // mail metadata - let mails_meta = self.mailbox.fetch_meta(&mails_uuid).await?; - - // fectch email body and transform to a state usable by select - let mails = if need_body { - let mut iter = mails - .into_iter() - .zip(mails_meta.into_iter()) - .map(|((i, uid, uuid), meta)| async move { - let body = self.mailbox.fetch_full(uuid, &meta.message_key).await?; - Ok::<_, anyhow::Error>((i, uid, uuid, meta, Some(body))) - }) - .collect::>(); - let mut mails = vec![]; - while let Some(m) = iter.next().await { - mails.push(m?); - } - mails - } else { - mails - .into_iter() - .zip(mails_meta.into_iter()) - .map(|((i, uid, uuid), meta)| (i, uid, uuid, meta, None)) - .collect::>() - }; - - // add flags - let mails = mails.into_iter().filter_map(|(i, uid, uuid, meta, body)| - self - .known_state - .table - .get(&uuid) - .map(|(_uid2, flags)| (i, uid, uuid, meta, flags, body))) - .collect::>(); - - // parse email body / headers - let fetched = match &body { - Some(m) => { - FetchedMail::Full(eml_codec::parse_message(m).or(Err(anyhow!("Invalid mail body")))?.1) - } - None => { - FetchedMail::Partial(eml_codec::parse_imf(&meta.headers).or(Err(anyhow!("Invalid mail headers")))?.1) - } - }; - - // do the logic - select_mail_fragments(fetch_attrs, mails) - } + // Fetch various metadata about the email selection + let selection = MailSelectionBuilder::new(ap.need_body(), mail_count); + selection.with_mail_identifiers(self.get_mail_ids(sequence_set, *is_uid_fetch)?); + selection.with_metadata(self.mailbox.fetch_meta(&selection.uuids()).await?); + selection.with_flags(self.known_state.table.get(&selection.uuids()).map(|(_, flags)| flags).collect::>()); - todo!(); + // Asynchronously fetch full bodies (if needed) + let future_bodies = selection.bodies_to_collect().iter().map(|bi| async move { + let body = self.mailbox.fetch_full(bi.msg_uuid, bi.msg_key).await?; + Ok::<_, anyhow::Error>(bi, body) + }).collect::>(); + let bodies = join_all(future_bodies).await.into_iter().collect::, _>>()?; + selection.with_bodies(bodies); - let mut ret = vec![]; - for (i, uid, uuid, meta, flags, body) in mails { - let mut attributes = vec![]; + /* while let Some(m) = future_bodies.next().await { + let (bi, body) = m?; + selection.with_body(bi.pos, body); + }*/ - for attr in fetch_attrs.iter() { - match attr { - FetchAttribute::Uid => attributes.push(MessageAttribute::Uid(uid)), - FetchAttribute::Flags => { - attributes.push(MessageAttribute::Flags( - flags.iter().filter_map(|f| string_to_flag(f)).collect(), - )); - } - FetchAttribute::Rfc822Size => { - attributes.push(MessageAttribute::Rfc822Size(meta.rfc822_size as u32)) - } - FetchAttribute::Rfc822Header => { - attributes.push(MessageAttribute::Rfc822Header(NString( - meta.headers.to_vec().try_into().ok().map(IString::Literal), - ))) - } - FetchAttribute::Rfc822Text => { - attributes.push(MessageAttribute::Rfc822Text(NString( - fetched.as_full()?.raw_body.try_into().ok().map(IString::Literal), - ))); - } - FetchAttribute::Rfc822 => attributes.push(MessageAttribute::Rfc822(NString( - body.as_ref() - .expect(BODY_CHECK) - .clone() - .try_into() - .ok() - .map(IString::Literal), - ))), - FetchAttribute::Envelope => { - attributes.push(MessageAttribute::Envelope(message_envelope(fetched.imf()))) - } - FetchAttribute::Body => attributes.push(MessageAttribute::Body( - build_imap_email_struct(fetched.as_full()?.child.as_ref())?, - )), - FetchAttribute::BodyStructure => attributes.push(MessageAttribute::Body( - build_imap_email_struct(fetched.as_full()?.child.as_ref())?, - )), + // Build mail selection views + let views = selection.build()?; - // maps to BODY[
]<> and BODY.PEEK[
]<> - // peek does not implicitly set the \Seen flag - // - // eg. BODY[HEADER.FIELDS (DATE FROM)] - // eg. BODY[]<0.2048> - FetchAttribute::BodyExt { - section, - partial, - peek, - } => { - // @TODO Add missing section specifiers - - // Extract message section - match get_message_section(fetched.as_full()?, section) { - Ok(text) => { - let seen_flag = Flag::Seen.to_string(); - if !peek && !flags.iter().any(|x| *x == seen_flag) { - // Add \Seen flag - self.mailbox.add_flags(uuid, &[seen_flag]).await?; - } - - // Handle <> which cut the message byts - let (text, origin) = match partial { - Some((begin, len)) => { - if *begin as usize > text.len() { - (&[][..], Some(*begin)) - } else if (*begin + len.get()) as usize >= text.len() { - (&text[*begin as usize..], Some(*begin)) - } else { - ( - &text[*begin as usize - ..(*begin + len.get()) as usize], - Some(*begin), - ) - } - } - None => (&text[..], None), - }; - - let data = - NString(text.to_vec().try_into().ok().map(IString::Literal)); - attributes.push(MessageAttribute::BodyExt { - section: section.clone(), - origin, - data, - }) - } - Err(e) => { - tracing::error!( - "Could not get section {:?} of message {}: {}", - section, - uuid, - e - ); - } - } - } - FetchAttribute::InternalDate => { - let dt = Utc.fix().timestamp_opt(i64::try_from(meta.internaldate / 1000)?, 0).earliest().ok_or(anyhow!("Unable to parse internal date"))?; - attributes.push(MessageAttribute::InternalDate(MyDateTime(dt))); - } - } + // Filter views to build the result + let mut ret = Vec::with_capacity(mail_count); + for mail_view in views.iter() { + let maybe_body = mail_view.filter(&ap)?; + if let Some(body) = maybe_body { + ret.push(body) } - - ret.push(Body::Data(Data::Fetch { - seq_or_uid: i, - attributes, - })); } + // Register seen flags + let future_flags = views.iter().filter(|mv| mv.add_seen).map(|mv| async move { + let seen_flag = Flag::Seen.to_string(); + self.mailbox.add_flags(mv.mi.uuid, &[seen_flag]).await?; + Ok::<_, anyhow::Error>(()) + }).collect::>(); + join_all(future_flags).await.iter().find(Result::is_err).unwrap_or(Ok(()))?; + Ok(ret) } @@ -536,7 +570,7 @@ impl MailboxView { &self, sequence_set: &SequenceSet, by_uid: bool, - ) -> Result> { + ) -> Result> { let mail_vec = self .known_state .idx_by_uid @@ -561,7 +595,7 @@ impl MailboxView { } if let Some(mail) = mail_vec.get(i) { if mail.0 == uid { - mails.push((NonZeroU32::try_from(i as u32 + 1).unwrap(), mail.0, mail.1)); + mails.push(MailIdentifiers { id: NonZeroU32::try_from(i as u32 + 1).unwrap(), uid: mail.0, uuid: mail.1 }); } } else { break; @@ -578,7 +612,7 @@ impl MailboxView { for i in sequence_set.iter(iter_strat) { if let Some(mail) = mail_vec.get(i.get() as usize - 1) { - mails.push((i, mail.0, mail.1)); + mails.push(MailIdentifiers { id: i, uid: mail.0, uuid: mail.1 }); } else { bail!("No such mail: {}", i); } From b444ef7ef3d6e0e57726130e3b89a676cdcd6245 Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Tue, 10 Oct 2023 17:59:34 +0200 Subject: [PATCH 5/6] finally code that build --- src/imap/mailbox_view.rs | 230 +++++++++++++++++++-------------------- 1 file changed, 109 insertions(+), 121 deletions(-) diff --git a/src/imap/mailbox_view.rs b/src/imap/mailbox_view.rs index 1378378..ecc0208 100644 --- a/src/imap/mailbox_view.rs +++ b/src/imap/mailbox_view.rs @@ -7,7 +7,6 @@ use anyhow::{anyhow, bail, Error, Result}; use boitalettres::proto::res::body::Data as Body; use chrono::{Offset, TimeZone, Utc}; -use futures::future::join_all; use futures::stream::{FuturesOrdered, StreamExt}; use imap_codec::types::address::Address; @@ -75,7 +74,7 @@ impl AttributesProxy { }; // Handle uids - if *is_uid_fetch && !fetch_attrs.contains(&FetchAttribute::Uid) { + if is_uid_fetch && !fetch_attrs.contains(&FetchAttribute::Uid) { fetch_attrs.push(FetchAttribute::Uid); } @@ -101,11 +100,18 @@ pub struct MailIdentifiers { uid: ImapUid, uuid: UniqueIdent, } +struct MailIdentifiersList(Vec); + +impl MailIdentifiersList { + fn uuids(&self) -> Vec { + self.0.iter().map(|mi| mi.uuid).collect() + } +} pub struct MailView<'a> { ids: &'a MailIdentifiers, meta: &'a MailMeta, - flags: &'a Vec, + flags: &'a Vec, content: FetchedMail<'a>, add_seen: bool } @@ -141,7 +147,7 @@ impl<'a> MailView<'a> { } fn envelope(&self) -> MessageAttribute { - message_envelope(self.content.imf()) + MessageAttribute::Envelope(message_envelope(self.content.imf())) } fn body(&self) -> Result { @@ -160,38 +166,28 @@ impl<'a> MailView<'a> { /// peek does not implicitly set the \Seen flag /// eg. BODY[HEADER.FIELDS (DATE FROM)] /// eg. BODY[]<0.2048> - fn body_ext(&mut self, section: Option, partial: Option<(u32, NonZeroU32)>, peek: bool) -> Result> { + fn body_ext(&mut self, section: &Option, partial: &Option<(u32, NonZeroU32)>, peek: &bool) -> Result { // Extract message section - match get_message_section(self.content.as_full()?, section) { - Ok(text) => { - let seen_flag = Flag::Seen.to_string(); - if !peek && !self.flags.iter().any(|x| *x == seen_flag) { - // Add \Seen flag - //self.mailbox.add_flags(uuid, &[seen_flag]).await?; - self.add_seen = true; - } + let text = get_message_section(self.content.as_full()?, section)?; - // Handle <> which cut the message bytes - let (text, origin) = apply_partial(partial, text); - - let data = NString(text.to_vec().try_into().ok().map(IString::Literal)); - - return Ok(Some(MessageAttribute::BodyExt { - section: section.clone(), - origin, - data, - })) - } - Err(e) => { - tracing::error!( - "Could not get section {:?} of message {}: {}", - section, - self.mi.uuid, - e - ); - return Ok(None) - } + let seen_flag = Flag::Seen.to_string(); + if !peek && !self.flags.iter().any(|x| *x == seen_flag) { + // Add \Seen flag + //self.mailbox.add_flags(uuid, &[seen_flag]).await?; + self.add_seen = true; } + + // Handle <> which cut the message bytes + let (text, origin) = apply_partial(partial, &text); + + let data = NString(text.to_vec().try_into().ok().map(IString::Literal)); + + return Ok(MessageAttribute::BodyExt { + section: section.clone(), + origin, + data, + }) + } fn internal_date(&self) -> Result { @@ -199,45 +195,44 @@ impl<'a> MailView<'a> { Ok(MessageAttribute::InternalDate(MyDateTime(dt))) } - fn filter(&mut self, ap: &AttributesProxy) -> Result> { - let res_attrs = ap.attrs.iter().filter_map(|attr| match attr { - FetchAttribute::Uid => Some(self.uid()), - FetchAttribute::Flags => Some(self.flags()), - FetchAttribute::Rfc822Size => Some(self.rfc_822_size()), - FetchAttribute::Rfc822Header => Some(self.rfc_822_header()), - FetchAttribute::Rfc822Text => Some(self.rfc_822_text()?), - FetchAttribute::Rfc822 => Some(self.rfc822()?), - FetchAttribute::Envelope => Some(self.envelope()), - FetchAttribute::Body => Some(self.body()?), - FetchAttribute::BodyStructure => Some(self.body_structure()?), - FetchAttribute::BodyExt { section, partial, peek } => self.body_ext(section, partial, peek)?, - FetchAttribute::InternalDate => Some(self.internal_date()?) - }).collect::>(); + fn filter(&mut self, ap: &AttributesProxy) -> Result { + let res_attrs = ap.attrs.iter().map(|attr| match attr { + FetchAttribute::Uid => Ok(self.uid()), + FetchAttribute::Flags => Ok(self.flags()), + FetchAttribute::Rfc822Size => Ok(self.rfc_822_size()), + FetchAttribute::Rfc822Header => Ok(self.rfc_822_header()), + FetchAttribute::Rfc822Text => self.rfc_822_text(), + FetchAttribute::Rfc822 => self.rfc822(), + FetchAttribute::Envelope => Ok(self.envelope()), + FetchAttribute::Body => self.body(), + FetchAttribute::BodyStructure => self.body_structure(), + FetchAttribute::BodyExt { section, partial, peek } => self.body_ext(section, partial, peek), + FetchAttribute::InternalDate => self.internal_date(), + }).collect::, _>>()?; - Body::Data(Data::Fetch { + Ok(Body::Data(Data::Fetch { seq_or_uid: self.ids.i, - res_attrs, - }) + attributes: res_attrs, + })) } } -fn apply_partial(partial: Option<(u32, NonZeroU32)>, text: &[u8]) -> (&[u8], Option) { +fn apply_partial<'a>(partial: &'_ Option<(u32, NonZeroU32)>, text: &'a [u8]) -> (&'a [u8], Option) { match partial { Some((begin, len)) => { if *begin as usize > text.len() { (&[][..], Some(*begin)) - } else if (*begin + len.get()) as usize >= text.len() { + } else if (begin + len.get()) as usize >= text.len() { (&text[*begin as usize..], Some(*begin)) } else { ( - &text[*begin as usize..(*begin + len.get()) as usize], + &text[*begin as usize..(begin + len.get()) as usize], Some(*begin), ) } } None => (&text[..], None), - }; - + } } pub struct BodyIdentifier<'a> { @@ -252,8 +247,8 @@ pub struct MailSelectionBuilder<'a> { need_body: bool, mi: &'a [MailIdentifiers], meta: &'a [MailMeta], - flags: &'a [Vec], - bodies: Vec>, + flags: &'a [&'a Vec], + bodies: &'a [Vec], } impl<'a> MailSelectionBuilder<'a> { @@ -261,65 +256,59 @@ impl<'a> MailSelectionBuilder<'a> { Self { mail_count, need_body, - bodies: vec![0; mail_count], ..MailSelectionBuilder::default() } } - fn with_mail_identifiers(mut self, mi: &[MailIdentifiers]) -> Self { + fn with_mail_identifiers(&mut self, mi: &'a [MailIdentifiers]) -> &mut Self { self.mi = mi; self } - fn uuids(&self) -> Vec<&UniqueIdent> { - self.mi.iter().map(|i| i.uuid ).collect::>() - } - - fn with_metadata(mut self, meta: &[MailMeta]) -> Self { + fn with_metadata(&mut self, meta: &'a [MailMeta]) -> &mut Self { self.meta = meta; self } - fn with_flags(mut self, flags: &[Vec]) -> Self { + fn with_flags(&mut self, flags: &'a [&'a Vec]) -> &mut Self { self.flags = flags; self } fn bodies_to_collect(&self) -> Vec { - if !self.attrs.need_body() { + if !self.need_body { return vec![] } - zip(self.mi, self.meta).as_ref().iter().enumerate().map(|(i , (mi, k))| BodyIdentifier { i, mi, k }).collect::>() + zip(self.mi, self.meta).map(|(mi, meta)| BodyIdentifier { msg_uuid: &mi.uuid, msg_key: &meta.message_key }).collect::>() } - fn with_bodies(mut self, rbodies: &[&'a [u8]]) -> Self { - for rb in rbodies.iter() { - let (_, p) = eml_codec::parse_message(&rb).or(Err(anyhow!("Invalid mail body")))?; - self.bodies.push(FetchedMail::Full(p)); - } - + fn with_bodies(&mut self, rbodies: &'a [Vec]) -> &mut Self { + self.bodies = rbodies; self } - fn build(self) -> Result>> { - if !self.need_body && self.bodies.len() == 0 { + fn build(&self) -> Result>> { + let mut bodies = vec![]; + + if !self.need_body { for m in self.meta.iter() { - let (_, hdrs) = eml_codec::parse_imf(&self.meta.headers).or(Err(anyhow!("Invalid mail headers")))?; - self.bodies.push(FetchedMail::Partial(hdrs)); + let (_, hdrs) = eml_codec::parse_imf(&m.headers).or(Err(anyhow!("Invalid mail headers")))?; + bodies.push(FetchedMail::Partial(hdrs)); + } + } else { + for rb in self.bodies.iter() { + let (_, p) = eml_codec::parse_message(&rb).or(Err(anyhow!("Invalid mail body")))?; + bodies.push(FetchedMail::Full(p)); } } - if self.mi.len() != self.mail_count && self.meta.len() != self.mail_count || self.flags.len() != self.mail_count || self.bodies.len() != self.mail_count { + if self.mi.len() != self.mail_count && self.meta.len() != self.mail_count || self.flags.len() != self.mail_count || bodies.len() != self.mail_count { return Err(anyhow!("Can't build a mail view selection as parts were not correctly registered into the builder.")) } - let views = Vec::with_capacity(self.mail_count); - for (ids, meta, flags, content) in zip(self.mi, self.meta, self.flags, self.bodies) { - let mv = MailView { ids, meta, flags, content }; - views.push(mv); - } - - return Ok(views) + Ok(zip(self.mi, zip(self.meta, zip(self.flags, bodies))) + .map(|(ids, (meta, (flags, content)))| MailView { ids, meta, flags, content, add_seen: false }) + .collect()) } } @@ -445,16 +434,16 @@ impl MailboxView { let flags = flags.iter().map(|x| x.to_string()).collect::>(); let mails = self.get_mail_ids(sequence_set, *is_uid_store)?; - for (_i, _uid, uuid) in mails.iter() { + for mi in mails.iter() { match kind { StoreType::Add => { - self.mailbox.add_flags(*uuid, &flags[..]).await?; + self.mailbox.add_flags(mi.uuid, &flags[..]).await?; } StoreType::Remove => { - self.mailbox.del_flags(*uuid, &flags[..]).await?; + self.mailbox.del_flags(mi.uuid, &flags[..]).await?; } StoreType::Replace => { - self.mailbox.set_flags(*uuid, &flags[..]).await?; + self.mailbox.set_flags(mi.uuid, &flags[..]).await?; } } } @@ -490,19 +479,19 @@ impl MailboxView { let mails = self.get_mail_ids(sequence_set, *is_uid_copy)?; let mut new_uuids = vec![]; - for (_i, _uid, uuid) in mails.iter() { - new_uuids.push(to.copy_from(&self.mailbox, *uuid).await?); + for mi in mails.iter() { + new_uuids.push(to.copy_from(&self.mailbox, mi.uuid).await?); } let mut ret = vec![]; let to_state = to.current_uid_index().await; - for ((_i, uid, _uuid), new_uuid) in mails.iter().zip(new_uuids.iter()) { + for (mi, new_uuid) in mails.iter().zip(new_uuids.iter()) { let dest_uid = to_state .table .get(new_uuid) .ok_or(anyhow!("copied mail not in destination mailbox"))? .0; - ret.push((*uid, dest_uid)); + ret.push((mi.uid, dest_uid)); } Ok((to_state.uidvalidity, ret)) @@ -518,46 +507,45 @@ impl MailboxView { ) -> Result> { let ap = AttributesProxy::new(attributes, *is_uid_fetch); - let mail_count = sequence_set.iter().count(); - // Fetch various metadata about the email selection - let selection = MailSelectionBuilder::new(ap.need_body(), mail_count); - selection.with_mail_identifiers(self.get_mail_ids(sequence_set, *is_uid_fetch)?); - selection.with_metadata(self.mailbox.fetch_meta(&selection.uuids()).await?); - selection.with_flags(self.known_state.table.get(&selection.uuids()).map(|(_, flags)| flags).collect::>()); + // Prepare data + let mids = MailIdentifiersList(self.get_mail_ids(sequence_set, *is_uid_fetch)?); + let mail_count = mids.0.len(); + let uuids = mids.uuids(); + let meta = self.mailbox.fetch_meta(&uuids).await?; + let flags = uuids.iter().map(|uuid| self.known_state.table.get(uuid).map(|(_uuid, f)| f).ok_or(anyhow!("missing email from the flag table"))).collect::, _>>()?; + + // Start filling data to build the view + let mut selection = MailSelectionBuilder::new(ap.need_body(), mail_count); + selection + .with_mail_identifiers(&mids.0) + .with_metadata(&meta) + .with_flags(&flags); // Asynchronously fetch full bodies (if needed) - let future_bodies = selection.bodies_to_collect().iter().map(|bi| async move { - let body = self.mailbox.fetch_full(bi.msg_uuid, bi.msg_key).await?; - Ok::<_, anyhow::Error>(bi, body) + let btc = selection.bodies_to_collect(); + let future_bodies = btc.iter().map(|bi| async move { + let body = self.mailbox.fetch_full(*bi.msg_uuid, bi.msg_key).await?; + Ok::<_, anyhow::Error>(body) }).collect::>(); - let bodies = join_all(future_bodies).await.into_iter().collect::, _>>()?; - selection.with_bodies(bodies); + let bodies = future_bodies.collect::>().await.into_iter().collect::, _>>()?; - /* while let Some(m) = future_bodies.next().await { - let (bi, body) = m?; - selection.with_body(bi.pos, body); - }*/ + // Add bodies + selection.with_bodies(bodies.as_slice()); // Build mail selection views - let views = selection.build()?; + let mut views = selection.build()?; // Filter views to build the result - let mut ret = Vec::with_capacity(mail_count); - for mail_view in views.iter() { - let maybe_body = mail_view.filter(&ap)?; - if let Some(body) = maybe_body { - ret.push(body) - } - } + let ret = views.iter_mut().filter_map(|mv| mv.filter(&ap).ok()).collect::>(); // Register seen flags let future_flags = views.iter().filter(|mv| mv.add_seen).map(|mv| async move { let seen_flag = Flag::Seen.to_string(); - self.mailbox.add_flags(mv.mi.uuid, &[seen_flag]).await?; + self.mailbox.add_flags(mv.ids.uuid, &[seen_flag]).await?; Ok::<_, anyhow::Error>(()) }).collect::>(); - join_all(future_flags).await.iter().find(Result::is_err).unwrap_or(Ok(()))?; + future_flags.collect::>().await.into_iter().collect::>()?; Ok(ret) } @@ -595,7 +583,7 @@ impl MailboxView { } if let Some(mail) = mail_vec.get(i) { if mail.0 == uid { - mails.push(MailIdentifiers { id: NonZeroU32::try_from(i as u32 + 1).unwrap(), uid: mail.0, uuid: mail.1 }); + mails.push(MailIdentifiers { i: NonZeroU32::try_from(i as u32 + 1).unwrap(), uid: mail.0, uuid: mail.1 }); } } else { break; @@ -612,7 +600,7 @@ impl MailboxView { for i in sequence_set.iter(iter_strat) { if let Some(mail) = mail_vec.get(i.get() as usize - 1) { - mails.push(MailIdentifiers { id: i, uid: mail.0, uuid: mail.1 }); + mails.push(MailIdentifiers { i, uid: mail.0, uuid: mail.1 }); } else { bail!("No such mail: {}", i); } @@ -784,7 +772,7 @@ fn message_envelope(msg: &imf::Imf) -> Envelope { } else { convert_addresses(&msg.reply_to) }, - from: from, + from, to: convert_addresses(&msg.to), cc: convert_addresses(&msg.cc), bcc: convert_addresses(&msg.bcc), From a1b7ca17c0badff4c983f3738531f3aa9b0fb977 Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Thu, 12 Oct 2023 12:21:59 +0200 Subject: [PATCH 6/6] basic body ext testing + format --- src/bayou.rs | 4 +- src/imap/mailbox_view.rs | 486 ++++++++++++++++++++++++++++----------- src/login/mod.rs | 16 +- src/mail/incoming.rs | 8 +- src/mail/mailbox.rs | 6 +- src/mail/user.rs | 21 +- 6 files changed, 379 insertions(+), 162 deletions(-) diff --git a/src/bayou.rs b/src/bayou.rs index cbfb414..9f70017 100644 --- a/src/bayou.rs +++ b/src/bayou.rs @@ -384,7 +384,7 @@ impl Bayou { let cryptoblob = seal_serialize(&state_cp, &self.key)?; debug!("(cp) checkpoint body length: {}", cryptoblob.len()); - let por = PutObjectRequest{ + let por = PutObjectRequest { bucket: self.bucket.clone(), key: format!("{}/checkpoint/{}", self.path, ts_cp.to_string()), body: Some(cryptoblob.into()), @@ -437,7 +437,7 @@ impl Bayou { async fn list_checkpoints(&self) -> Result> { let prefix = format!("{}/checkpoint/", self.path); - let lor = ListObjectsV2Request{ + let lor = ListObjectsV2Request { bucket: self.bucket.clone(), max_keys: Some(1000), prefix: Some(prefix.clone()), diff --git a/src/imap/mailbox_view.rs b/src/imap/mailbox_view.rs index ecc0208..99069e2 100644 --- a/src/imap/mailbox_view.rs +++ b/src/imap/mailbox_view.rs @@ -1,7 +1,7 @@ use std::borrow::Cow; +use std::iter::zip; use std::num::NonZeroU32; use std::sync::Arc; -use std::iter::zip; use anyhow::{anyhow, bail, Error, Result}; use boitalettres::proto::res::body::Data as Body; @@ -22,15 +22,13 @@ use imap_codec::types::response::{Code, Data, MessageAttribute, Status}; use imap_codec::types::sequence::{self, SequenceSet}; use eml_codec::{ - header, - imf, - part::{AnyPart, composite::Message}, + header, imf, mime, mime::r#type::Deductible, - mime, + part::{composite::Message, AnyPart}, }; use crate::cryptoblob::Key; -use crate::mail::mailbox::{Mailbox, MailMeta}; +use crate::mail::mailbox::{MailMeta, Mailbox}; use crate::mail::uidindex::{ImapUid, ImapUidvalidity, UidIndex}; use crate::mail::unique_ident::UniqueIdent; @@ -44,26 +42,42 @@ const DEFAULT_FLAGS: [Flag; 5] = [ enum FetchedMail<'a> { Partial(imf::Imf<'a>), - Full(Message<'a>), + Full(AnyPart<'a>), } impl<'a> FetchedMail<'a> { - fn as_full(&self) -> Result<&Message<'a>> { + fn new_from_message(msg: Message<'a>) -> Self { + FetchedMail::Full(AnyPart::Msg(msg)) + } + + /*fn new_from_header(hdr: imf::Imf<'a>) -> Self { + FetchedMail::Partial(hdr) + }*/ + + fn as_anypart(&self) -> Result<&AnyPart<'a>> { match self { FetchedMail::Full(x) => Ok(&x), - _ => bail!("The full message must be fetched, not only its headers: it's a logic error"), + _ => bail!("The full message must be fetched, not only its headers"), + } + } + + fn as_full(&self) -> Result<&Message<'a>> { + match self { + FetchedMail::Full(AnyPart::Msg(x)) => Ok(&x), + _ => bail!("The full message must be fetched, not only its headers AND it must be an AnyPart::Msg."), } } fn imf(&self) -> &imf::Imf<'a> { match self { - FetchedMail::Full(x) => &x.imf, + FetchedMail::Full(AnyPart::Msg(x)) => &x.imf, FetchedMail::Partial(x) => &x, + _ => panic!("Can't contain AnyPart that is not a message"), } } } pub struct AttributesProxy { - attrs: Vec + attrs: Vec, } impl AttributesProxy { fn new(attrs: &MacroOrFetchAttributes, is_uid_fetch: bool) -> Self { @@ -112,8 +126,8 @@ pub struct MailView<'a> { ids: &'a MailIdentifiers, meta: &'a MailMeta, flags: &'a Vec, - content: FetchedMail<'a>, - add_seen: bool + content: FetchedMail<'a>, + add_seen: bool, } impl<'a> MailView<'a> { @@ -123,7 +137,10 @@ impl<'a> MailView<'a> { fn flags(&self) -> MessageAttribute { MessageAttribute::Flags( - self.flags.iter().filter_map(|f| string_to_flag(f)).collect(), + self.flags + .iter() + .filter_map(|f| string_to_flag(f)) + .collect(), ) } @@ -132,18 +149,37 @@ impl<'a> MailView<'a> { } fn rfc_822_header(&self) -> MessageAttribute { - MessageAttribute::Rfc822Header(NString(self.meta.headers.to_vec().try_into().ok().map(IString::Literal))) + MessageAttribute::Rfc822Header(NString( + self.meta + .headers + .to_vec() + .try_into() + .ok() + .map(IString::Literal), + )) } fn rfc_822_text(&self) -> Result { Ok(MessageAttribute::Rfc822Text(NString( - self.content.as_full()?.raw_body.try_into().ok().map(IString::Literal), + self.content + .as_full()? + .raw_body + .try_into() + .ok() + .map(IString::Literal), ))) } fn rfc822(&self) -> Result { Ok(MessageAttribute::Rfc822(NString( - self.content.as_full()?.raw_body.clone().try_into().ok().map(IString::Literal)))) + self.content + .as_full()? + .raw_body + .clone() + .try_into() + .ok() + .map(IString::Literal), + ))) } fn envelope(&self) -> MessageAttribute { @@ -151,24 +187,29 @@ impl<'a> MailView<'a> { } fn body(&self) -> Result { - Ok(MessageAttribute::Body( - build_imap_email_struct(self.content.as_full()?.child.as_ref())?, - )) + Ok(MessageAttribute::Body(build_imap_email_struct( + self.content.as_full()?.child.as_ref(), + )?)) } fn body_structure(&self) -> Result { - Ok(MessageAttribute::Body( - build_imap_email_struct(self.content.as_full()?.child.as_ref())?, - )) + Ok(MessageAttribute::Body(build_imap_email_struct( + self.content.as_full()?.child.as_ref(), + )?)) } /// maps to BODY[
]<> and BODY.PEEK[
]<> /// peek does not implicitly set the \Seen flag /// eg. BODY[HEADER.FIELDS (DATE FROM)] /// eg. BODY[]<0.2048> - fn body_ext(&mut self, section: &Option, partial: &Option<(u32, NonZeroU32)>, peek: &bool) -> Result { + fn body_ext( + &mut self, + section: &Option, + partial: &Option<(u32, NonZeroU32)>, + peek: &bool, + ) -> Result { // Extract message section - let text = get_message_section(self.content.as_full()?, section)?; + let text = get_message_section(self.content.as_anypart()?, section)?; let seen_flag = Flag::Seen.to_string(); if !peek && !self.flags.iter().any(|x| *x == seen_flag) { @@ -186,29 +227,40 @@ impl<'a> MailView<'a> { section: section.clone(), origin, data, - }) - + }); } fn internal_date(&self) -> Result { - let dt = Utc.fix().timestamp_opt(i64::try_from(self.meta.internaldate / 1000)?, 0).earliest().ok_or(anyhow!("Unable to parse internal date"))?; + let dt = Utc + .fix() + .timestamp_opt(i64::try_from(self.meta.internaldate / 1000)?, 0) + .earliest() + .ok_or(anyhow!("Unable to parse internal date"))?; Ok(MessageAttribute::InternalDate(MyDateTime(dt))) } fn filter(&mut self, ap: &AttributesProxy) -> Result { - let res_attrs = ap.attrs.iter().map(|attr| match attr { - FetchAttribute::Uid => Ok(self.uid()), - FetchAttribute::Flags => Ok(self.flags()), - FetchAttribute::Rfc822Size => Ok(self.rfc_822_size()), - FetchAttribute::Rfc822Header => Ok(self.rfc_822_header()), - FetchAttribute::Rfc822Text => self.rfc_822_text(), - FetchAttribute::Rfc822 => self.rfc822(), - FetchAttribute::Envelope => Ok(self.envelope()), - FetchAttribute::Body => self.body(), - FetchAttribute::BodyStructure => self.body_structure(), - FetchAttribute::BodyExt { section, partial, peek } => self.body_ext(section, partial, peek), - FetchAttribute::InternalDate => self.internal_date(), - }).collect::, _>>()?; + let res_attrs = ap + .attrs + .iter() + .map(|attr| match attr { + FetchAttribute::Uid => Ok(self.uid()), + FetchAttribute::Flags => Ok(self.flags()), + FetchAttribute::Rfc822Size => Ok(self.rfc_822_size()), + FetchAttribute::Rfc822Header => Ok(self.rfc_822_header()), + FetchAttribute::Rfc822Text => self.rfc_822_text(), + FetchAttribute::Rfc822 => self.rfc822(), + FetchAttribute::Envelope => Ok(self.envelope()), + FetchAttribute::Body => self.body(), + FetchAttribute::BodyStructure => self.body_structure(), + FetchAttribute::BodyExt { + section, + partial, + peek, + } => self.body_ext(section, partial, peek), + FetchAttribute::InternalDate => self.internal_date(), + }) + .collect::, _>>()?; Ok(Body::Data(Data::Fetch { seq_or_uid: self.ids.i, @@ -217,7 +269,10 @@ impl<'a> MailView<'a> { } } -fn apply_partial<'a>(partial: &'_ Option<(u32, NonZeroU32)>, text: &'a [u8]) -> (&'a [u8], Option) { +fn apply_partial<'a>( + partial: &'_ Option<(u32, NonZeroU32)>, + text: &'a [u8], +) -> (&'a [u8], Option) { match partial { Some((begin, len)) => { if *begin as usize > text.len() { @@ -253,11 +308,11 @@ pub struct MailSelectionBuilder<'a> { impl<'a> MailSelectionBuilder<'a> { fn new(need_body: bool, mail_count: usize) -> Self { - Self { - mail_count, + Self { + mail_count, need_body, ..MailSelectionBuilder::default() - } + } } fn with_mail_identifiers(&mut self, mi: &'a [MailIdentifiers]) -> &mut Self { @@ -269,7 +324,7 @@ impl<'a> MailSelectionBuilder<'a> { self.meta = meta; self } - + fn with_flags(&mut self, flags: &'a [&'a Vec]) -> &mut Self { self.flags = flags; self @@ -277,9 +332,14 @@ impl<'a> MailSelectionBuilder<'a> { fn bodies_to_collect(&self) -> Vec { if !self.need_body { - return vec![] + return vec![]; } - zip(self.mi, self.meta).map(|(mi, meta)| BodyIdentifier { msg_uuid: &mi.uuid, msg_key: &meta.message_key }).collect::>() + zip(self.mi, self.meta) + .map(|(mi, meta)| BodyIdentifier { + msg_uuid: &mi.uuid, + msg_key: &meta.message_key, + }) + .collect::>() } fn with_bodies(&mut self, rbodies: &'a [Vec]) -> &mut Self { @@ -292,22 +352,32 @@ impl<'a> MailSelectionBuilder<'a> { if !self.need_body { for m in self.meta.iter() { - let (_, hdrs) = eml_codec::parse_imf(&m.headers).or(Err(anyhow!("Invalid mail headers")))?; + let (_, hdrs) = + eml_codec::parse_imf(&m.headers).or(Err(anyhow!("Invalid mail headers")))?; bodies.push(FetchedMail::Partial(hdrs)); } } else { for rb in self.bodies.iter() { let (_, p) = eml_codec::parse_message(&rb).or(Err(anyhow!("Invalid mail body")))?; - bodies.push(FetchedMail::Full(p)); + bodies.push(FetchedMail::new_from_message(p)); } } - if self.mi.len() != self.mail_count && self.meta.len() != self.mail_count || self.flags.len() != self.mail_count || bodies.len() != self.mail_count { - return Err(anyhow!("Can't build a mail view selection as parts were not correctly registered into the builder.")) + if self.mi.len() != self.mail_count && self.meta.len() != self.mail_count + || self.flags.len() != self.mail_count + || bodies.len() != self.mail_count + { + return Err(anyhow!("Can't build a mail view selection as parts were not correctly registered into the builder.")); } Ok(zip(self.mi, zip(self.meta, zip(self.flags, bodies))) - .map(|(ids, (meta, (flags, content)))| MailView { ids, meta, flags, content, add_seen: false }) + .map(|(ids, (meta, (flags, content)))| MailView { + ids, + meta, + flags, + content, + add_seen: false, + }) .collect()) } } @@ -505,7 +575,6 @@ impl MailboxView { attributes: &MacroOrFetchAttributes, is_uid_fetch: &bool, ) -> Result> { - let ap = AttributesProxy::new(attributes, *is_uid_fetch); // Prepare data @@ -513,22 +582,38 @@ impl MailboxView { let mail_count = mids.0.len(); let uuids = mids.uuids(); let meta = self.mailbox.fetch_meta(&uuids).await?; - let flags = uuids.iter().map(|uuid| self.known_state.table.get(uuid).map(|(_uuid, f)| f).ok_or(anyhow!("missing email from the flag table"))).collect::, _>>()?; + let flags = uuids + .iter() + .map(|uuid| { + self.known_state + .table + .get(uuid) + .map(|(_uuid, f)| f) + .ok_or(anyhow!("missing email from the flag table")) + }) + .collect::, _>>()?; // Start filling data to build the view let mut selection = MailSelectionBuilder::new(ap.need_body(), mail_count); selection - .with_mail_identifiers(&mids.0) - .with_metadata(&meta) - .with_flags(&flags); - + .with_mail_identifiers(&mids.0) + .with_metadata(&meta) + .with_flags(&flags); + // Asynchronously fetch full bodies (if needed) let btc = selection.bodies_to_collect(); - let future_bodies = btc.iter().map(|bi| async move { - let body = self.mailbox.fetch_full(*bi.msg_uuid, bi.msg_key).await?; - Ok::<_, anyhow::Error>(body) - }).collect::>(); - let bodies = future_bodies.collect::>().await.into_iter().collect::, _>>()?; + let future_bodies = btc + .iter() + .map(|bi| async move { + let body = self.mailbox.fetch_full(*bi.msg_uuid, bi.msg_key).await?; + Ok::<_, anyhow::Error>(body) + }) + .collect::>(); + let bodies = future_bodies + .collect::>() + .await + .into_iter() + .collect::, _>>()?; // Add bodies selection.with_bodies(bodies.as_slice()); @@ -537,15 +622,26 @@ impl MailboxView { let mut views = selection.build()?; // Filter views to build the result - let ret = views.iter_mut().filter_map(|mv| mv.filter(&ap).ok()).collect::>(); + let ret = views + .iter_mut() + .filter_map(|mv| mv.filter(&ap).ok()) + .collect::>(); // Register seen flags - let future_flags = views.iter().filter(|mv| mv.add_seen).map(|mv| async move { - let seen_flag = Flag::Seen.to_string(); - self.mailbox.add_flags(mv.ids.uuid, &[seen_flag]).await?; - Ok::<_, anyhow::Error>(()) - }).collect::>(); - future_flags.collect::>().await.into_iter().collect::>()?; + let future_flags = views + .iter() + .filter(|mv| mv.add_seen) + .map(|mv| async move { + let seen_flag = Flag::Seen.to_string(); + self.mailbox.add_flags(mv.ids.uuid, &[seen_flag]).await?; + Ok::<_, anyhow::Error>(()) + }) + .collect::>(); + future_flags + .collect::>() + .await + .into_iter() + .collect::>()?; Ok(ret) } @@ -583,7 +679,11 @@ impl MailboxView { } if let Some(mail) = mail_vec.get(i) { if mail.0 == uid { - mails.push(MailIdentifiers { i: NonZeroU32::try_from(i as u32 + 1).unwrap(), uid: mail.0, uuid: mail.1 }); + mails.push(MailIdentifiers { + i: NonZeroU32::try_from(i as u32 + 1).unwrap(), + uid: mail.0, + uuid: mail.1, + }); } } else { break; @@ -600,7 +700,11 @@ impl MailboxView { for i in sequence_set.iter(iter_strat) { if let Some(mail) = mail_vec.get(i.get() as usize - 1) { - mails.push(MailIdentifiers { i, uid: mail.0, uuid: mail.1 }); + mails.push(MailIdentifiers { + i, + uid: mail.0, + uuid: mail.1, + }); } else { bail!("No such mail: {}", i); } @@ -759,26 +863,39 @@ fn message_envelope(msg: &imf::Imf) -> Envelope { Envelope { date: NString( - msg.date.as_ref() + msg.date + .as_ref() .map(|d| IString::try_from(d.to_rfc3339()).unwrap()), ), subject: NString( - msg.subject.as_ref() + msg.subject + .as_ref() .map(|d| IString::try_from(d.to_string()).unwrap()), ), - sender: msg.sender.as_ref().map(|v| vec![convert_mbx(v)]).unwrap_or(from.clone()), + sender: msg + .sender + .as_ref() + .map(|v| vec![convert_mbx(v)]) + .unwrap_or(from.clone()), reply_to: if msg.reply_to.is_empty() { - from.clone() - } else { - convert_addresses(&msg.reply_to) - }, + from.clone() + } else { + convert_addresses(&msg.reply_to) + }, from, to: convert_addresses(&msg.to), cc: convert_addresses(&msg.cc), bcc: convert_addresses(&msg.bcc), - in_reply_to: NString(msg.in_reply_to.iter().next().map(|d| IString::try_from(d.to_string()).unwrap())), + in_reply_to: NString( + msg.in_reply_to + .iter() + .next() + .map(|d| IString::try_from(d.to_string()).unwrap()), + ), message_id: NString( - msg.msg_id.as_ref().map(|d| IString::try_from(d.to_string()).unwrap()), + msg.msg_id + .as_ref() + .map(|d| IString::try_from(d.to_string()).unwrap()), ), } } @@ -788,20 +905,28 @@ fn convert_addresses(addrlist: &Vec) -> Vec
{ for item in addrlist { match item { imf::address::AddressRef::Single(a) => acc.push(convert_mbx(a)), - imf::address::AddressRef::Many(l) => acc.extend(l.participants.iter().map(convert_mbx)) + imf::address::AddressRef::Many(l) => acc.extend(l.participants.iter().map(convert_mbx)), } } - return acc + return acc; } fn convert_mbx(addr: &imf::mailbox::MailboxRef) -> Address { Address::new( - NString(addr.name.as_ref().map(|x| IString::try_from(x.to_string()).unwrap())), + NString( + addr.name + .as_ref() + .map(|x| IString::try_from(x.to_string()).unwrap()), + ), // SMTP at-domain-list (source route) seems obsolete since at least 1991 // https://www.mhonarc.org/archive/html/ietf-822/1991-06/msg00060.html NString(None), - NString(Some(IString::try_from(addr.addrspec.local_part.to_string()).unwrap())), - NString(Some(IString::try_from(addr.addrspec.domain.to_string()).unwrap())), + NString(Some( + IString::try_from(addr.addrspec.local_part.to_string()).unwrap(), + )), + NString(Some( + IString::try_from(addr.addrspec.domain.to_string()).unwrap(), + )), ) } @@ -824,10 +949,12 @@ fn build_imap_email_struct<'a>(part: &AnyPart<'a>) -> Result { match part { AnyPart::Mult(x) => { let itype = &x.mime.interpreted_type; - let subtype = IString::try_from(itype.subtype.to_string()).unwrap_or(unchecked_istring("alternative")); + let subtype = IString::try_from(itype.subtype.to_string()) + .unwrap_or(unchecked_istring("alternative")); Ok(BodyStructure::Multi { - bodies: x.children + bodies: x + .children .iter() .filter_map(|inner| build_imap_email_struct(&inner).ok()) .collect(), @@ -845,18 +972,19 @@ fn build_imap_email_struct<'a>(part: &AnyPart<'a>) -> Result { AnyPart::Txt(x) => { let mut basic = basic_fields(&x.mime.fields, x.body.len())?; - // Get the interpreted content type, set it + // Get the interpreted content type, set it let itype = match &x.mime.interpreted_type { - Deductible::Inferred(v) | Deductible::Explicit(v) => v + Deductible::Inferred(v) | Deductible::Explicit(v) => v, }; - let subtype = IString::try_from(itype.subtype.to_string()).unwrap_or(unchecked_istring("plain")); + let subtype = + IString::try_from(itype.subtype.to_string()).unwrap_or(unchecked_istring("plain")); // Add charset to the list of parameters if we know it has been inferred as it will be // missing from the parsed content. if let Deductible::Inferred(charset) = &itype.charset { basic.parameter_list.push(( unchecked_istring("charset"), - IString::try_from(charset.to_string()).unwrap_or(unchecked_istring("us-ascii")) + IString::try_from(charset.to_string()).unwrap_or(unchecked_istring("us-ascii")), )); } @@ -874,15 +1002,21 @@ fn build_imap_email_struct<'a>(part: &AnyPart<'a>) -> Result { AnyPart::Bin(x) => { let basic = basic_fields(&x.mime.fields, x.body.len())?; - let default = mime::r#type::NaiveType { main: &b"application"[..], sub: &b"octet-stream"[..], params: vec![] }; + let default = mime::r#type::NaiveType { + main: &b"application"[..], + sub: &b"octet-stream"[..], + params: vec![], + }; let ct = x.mime.fields.ctype.as_ref().unwrap_or(&default); - let type_ = IString::try_from(String::from_utf8_lossy(ct.main).to_string()) - .or(Err(anyhow!("Unable to build IString from given Content-Type type given")))?; + let type_ = IString::try_from(String::from_utf8_lossy(ct.main).to_string()).or(Err( + anyhow!("Unable to build IString from given Content-Type type given"), + ))?; - - let subtype = IString::try_from(String::from_utf8_lossy(ct.sub).to_string()) - .or(Err(anyhow!("Unable to build IString from given Content-Type subtype given")))?; + let subtype = + IString::try_from(String::from_utf8_lossy(ct.sub).to_string()).or(Err(anyhow!( + "Unable to build IString from given Content-Type subtype given" + )))?; Ok(BodyStructure::Single { body: FetchBody { @@ -911,7 +1045,8 @@ fn build_imap_email_struct<'a>(part: &AnyPart<'a>) -> Result { } fn nol(input: &[u8]) -> u32 { - input.iter() + input + .iter() .filter(|x| **x == b'\n') .count() .try_into() @@ -925,13 +1060,22 @@ fn unchecked_istring(s: &'static str) -> IString { } fn basic_fields(m: &mime::NaiveMIME, sz: usize) -> Result { - let parameter_list = m.ctype + let parameter_list = m + .ctype .as_ref() - .map(|x| x.params.iter() - .map(|p| (IString::try_from(String::from_utf8_lossy(p.name).to_string()), IString::try_from(p.value.to_string()))) - .filter(|(k, v)| k.is_ok() && v.is_ok()) - .map(|(k, v)| (k.unwrap(), v.unwrap())) - .collect()) + .map(|x| { + x.params + .iter() + .map(|p| { + ( + IString::try_from(String::from_utf8_lossy(p.name).to_string()), + IString::try_from(p.value.to_string()), + ) + }) + .filter(|(k, v)| k.is_ok() && v.is_ok()) + .map(|(k, v)| (k.unwrap(), v.unwrap())) + .collect() + }) .unwrap_or(vec![]); Ok(BasicFields { @@ -939,17 +1083,18 @@ fn basic_fields(m: &mime::NaiveMIME, sz: usize) -> Result { id: NString( m.id.as_ref() .and_then(|ci| IString::try_from(ci.to_string()).ok()), - ), + ), description: NString( - m.description.as_ref() + m.description + .as_ref() .and_then(|cd| IString::try_from(cd.to_string()).ok()), ), content_transfer_encoding: match m.transfer_encoding { - mime::mechanism::Mechanism::_8Bit => unchecked_istring("8bit"), - mime::mechanism::Mechanism::Binary => unchecked_istring("binary"), - mime::mechanism::Mechanism::QuotedPrintable => unchecked_istring("quoted-printable"), - mime::mechanism::Mechanism::Base64 => unchecked_istring("base64"), - _ => unchecked_istring("7bit"), + mime::mechanism::Mechanism::_8Bit => unchecked_istring("8bit"), + mime::mechanism::Mechanism::Binary => unchecked_istring("binary"), + mime::mechanism::Mechanism::QuotedPrintable => unchecked_istring("quoted-printable"), + mime::mechanism::Mechanism::Base64 => unchecked_istring("base64"), + _ => unchecked_istring("7bit"), }, // @FIXME we can't compute the size of the message currently... size: u32::try_from(sz)?, @@ -983,25 +1128,34 @@ fn basic_fields(m: &mime::NaiveMIME, sz: usize) -> Result { /// 4.2.2.2 TEXT/RICHTEXT /// ``` fn get_message_section<'a>( - parsed: &'a Message<'a>, + parsed: &'a AnyPart<'a>, section: &Option, ) -> Result> { + let msg = parsed + .as_message() + .ok_or(anyhow!("Part must be a message"))?; match section { - Some(FetchSection::Text(None)) => { - Ok(parsed.raw_body.into()) - } + Some(FetchSection::Text(None)) => Ok(msg.raw_body.into()), Some(FetchSection::Text(Some(part))) => { - map_subpart(parsed.child.as_ref(), part.0.as_slice(), |part_msg| { - Ok(part_msg.as_message().ok_or(Error::msg("Not a message/rfc822 part while expected by request (xxx.TEXT)"))? + map_subpart(parsed, part.0.as_slice(), |part_msg| { + Ok(part_msg + .as_message() + .ok_or(Error::msg( + "Not a message/rfc822 part while expected by request (TEXT)", + ))? .raw_body .into()) }) } Some(FetchSection::Header(part)) => map_subpart( - parsed.child.as_ref(), + parsed, part.as_ref().map(|p| p.0.as_slice()).unwrap_or(&[]), |part_msg| { - Ok(part_msg.as_message().ok_or(Error::msg("Not a message/rfc822 part while expected by request (xxx.TEXT)"))? + Ok(part_msg + .as_message() + .ok_or(Error::msg( + "Not a message/rfc822 part while expected by request (HEADER)", + ))? .raw_headers .into()) }, @@ -1019,9 +1173,8 @@ fn get_message_section<'a>( }) .collect::>(); - map_subpart( - parsed.child.as_ref(), + parsed, part.as_ref().map(|p| p.0.as_slice()).unwrap_or(&[]), |part_msg| { let mut ret = vec![]; @@ -1030,11 +1183,7 @@ fn get_message_section<'a>( header::Field::Good(header::Kv2(k, v)) => (k, v), _ => continue, }; - if fields - .as_slice() - .iter() - .any(|x| (x == k) ^ invert) - { + if fields.as_slice().iter().any(|x| (x == k) ^ invert) { ret.extend(*k); ret.extend(b": "); ret.extend(*v); @@ -1046,7 +1195,7 @@ fn get_message_section<'a>( }, ) } - Some(FetchSection::Part(part)) => map_subpart(parsed.child.as_ref(), part.0.as_slice(), |part| { + Some(FetchSection::Part(part)) => map_subpart(parsed, part.0.as_slice(), |part| { let bytes = match &part { AnyPart::Txt(p) => p.body, AnyPart::Bin(p) => p.body, @@ -1055,7 +1204,7 @@ fn get_message_section<'a>( }; Ok(bytes.to_vec().into()) }), - Some(FetchSection::Mime(part)) => map_subpart(parsed.child.as_ref(), part.0.as_slice(), |part| { + Some(FetchSection::Mime(part)) => map_subpart(parsed, part.0.as_slice(), |part| { let bytes = match &part { AnyPart::Txt(p) => p.mime.fields.raw, AnyPart::Bin(p) => p.mime.fields.raw, @@ -1064,11 +1213,11 @@ fn get_message_section<'a>( }; Ok(bytes.to_vec().into()) }), - None => Ok(parsed.raw_part.into()), + None => Ok(msg.raw_part.into()), } } -/// Fetch a MIME SubPart +/// Fetch a MIME SubPart /// /// eg. FETCH BODY[4.2.2.1] -> [4, 2, 2, 1] fn map_subpart<'a, F, R>(part: &AnyPart<'a>, path: &[NonZeroU32], f: F) -> Result @@ -1094,9 +1243,72 @@ where #[cfg(test)] mod tests { use super::*; + use crate::cryptoblob; + use crate::mail::unique_ident; use imap_codec::codec::Encode; + use imap_codec::types::fetch_attributes::Section; use std::fs; + #[test] + fn mailview_body_ext() -> Result<()> { + let ap = AttributesProxy::new( + &MacroOrFetchAttributes::FetchAttributes(vec![FetchAttribute::BodyExt { + section: Some(Section::Header(None)), + partial: None, + peek: false, + }]), + false, + ); + + let flags = vec![]; + let key = cryptoblob::gen_key(); + let meta = MailMeta { + internaldate: 0u64, + headers: vec![], + message_key: key, + rfc822_size: 8usize, + }; + let ids = MailIdentifiers { + i: NonZeroU32::MIN, + uid: NonZeroU32::MIN, + uuid: unique_ident::gen_ident(), + }; + let rfc822 = b"Subject: hello\r\nFrom: a@a.a\r\nTo: b@b.b\r\nDate: Thu, 12 Oct 2023 08:45:28 +0000\r\n\r\nhello world"; + let content = FetchedMail::new_from_message(eml_codec::parse_message(rfc822)?.1); + + let mut mv = MailView { + ids: &ids, + content, + meta: &meta, + flags: &flags, + add_seen: false, + }; + let res_body = mv.filter(&ap)?; + + let fattr = match res_body { + Body::Data(Data::Fetch { + seq_or_uid: _seq, + attributes: attr, + }) => Ok(attr), + _ => Err(anyhow!("Not a fetch body")), + }?; + + assert_eq!(fattr.len(), 1); + + let (sec, _orig, _data) = match &fattr[0] { + MessageAttribute::BodyExt { + section, + origin, + data, + } => Ok((section, origin, data)), + _ => Err(anyhow!("not a body ext message attribute")), + }?; + + assert_eq!(sec.as_ref().unwrap(), &Section::Header(None)); + + Ok(()) + } + /// Future automated test. We use lossy utf8 conversion + lowercase everything, /// so this test might allow invalid results. But at least it allows us to quickly test a /// large variety of emails. @@ -1113,7 +1325,6 @@ mod tests { //"tests/emails/dxflrs/0005_mail-parser-readme", "tests/emails/dxflrs/0006_single-mime", "tests/emails/dxflrs/0007_raw_msg_in_rfc822", - /* *** (STRANGE) RFC *** */ //"tests/emails/rfc/000", // must return text/enriched, we return text/plain //"tests/emails/rfc/001", // does not recognize the multipart/external-body, breaks the @@ -1127,9 +1338,8 @@ mod tests { //"tests/emails/thirdparty/001", // same "tests/emails/thirdparty/002", // same - - /* *** LEGACY *** */ - //"tests/emails/legacy/000", // same issue with \r\r + /* *** LEGACY *** */ + //"tests/emails/legacy/000", // same issue with \r\r ]; for pref in prefixes.iter() { @@ -1139,7 +1349,9 @@ mod tests { let message = eml_codec::parse_message(&txt).unwrap().1; let mut resp = Vec::new(); - MessageAttribute::Body(build_imap_email_struct(&message.child)?).encode(&mut resp).unwrap(); + MessageAttribute::Body(build_imap_email_struct(&message.child)?) + .encode(&mut resp) + .unwrap(); let resp_str = String::from_utf8_lossy(&resp).to_lowercase(); diff --git a/src/login/mod.rs b/src/login/mod.rs index a204db4..3fab90a 100644 --- a/src/login/mod.rs +++ b/src/login/mod.rs @@ -7,7 +7,7 @@ use std::sync::Arc; use anyhow::{anyhow, bail, Context, Result}; use async_trait::async_trait; use k2v_client::{ - BatchInsertOp, BatchReadOp, CausalValue, CausalityToken, Filter, K2vClient, K2vValue + BatchInsertOp, BatchReadOp, CausalValue, CausalityToken, Filter, K2vClient, K2vValue, }; use rand::prelude::*; use rusoto_core::HttpClient; @@ -141,13 +141,13 @@ impl StorageCredentials { self.aws_secret_access_key.clone(), ); - let connector = hyper_rustls::HttpsConnectorBuilder::new() - .with_native_roots() - .https_or_http() - .enable_http1() - .enable_http2() - .build(); - let client = HttpClient::from_connector(connector); + let connector = hyper_rustls::HttpsConnectorBuilder::new() + .with_native_roots() + .https_or_http() + .enable_http1() + .enable_http2() + .build(); + let client = HttpClient::from_connector(connector); Ok(S3Client::new_with( client, diff --git a/src/mail/incoming.rs b/src/mail/incoming.rs index 290f1b3..b7d2f48 100644 --- a/src/mail/incoming.rs +++ b/src/mail/incoming.rs @@ -450,10 +450,10 @@ impl EncryptedMessage { let por = PutObjectRequest { bucket: creds.storage.bucket.clone(), key: format!("incoming/{}", gen_ident()), - metadata: Some( - [(MESSAGE_KEY.to_string(), key_header)] - .into_iter() - .collect::>(), + metadata: Some( + [(MESSAGE_KEY.to_string(), key_header)] + .into_iter() + .collect::>(), ), body: Some(self.encrypted_body.clone().into()), ..Default::default() diff --git a/src/mail/mailbox.rs b/src/mail/mailbox.rs index fe36a14..d92140d 100644 --- a/src/mail/mailbox.rs +++ b/src/mail/mailbox.rs @@ -369,7 +369,7 @@ impl MailboxInternal { // Save mail meta let meta = MailMeta { internaldate: now_msec(), - headers: mail.parsed.raw_headers.to_vec(), + headers: mail.parsed.raw_headers.to_vec(), message_key: message_key.clone(), rfc822_size: mail.raw.len(), }; @@ -400,7 +400,7 @@ impl MailboxInternal { futures::try_join!( async { // Delete mail body from S3 - let dor = DeleteObjectRequest{ + let dor = DeleteObjectRequest { bucket: self.bucket.clone(), key: format!("{}/{}", self.mail_path, ident), ..Default::default() @@ -461,7 +461,7 @@ impl MailboxInternal { futures::try_join!( async { // Copy mail body from S3 - let cor = CopyObjectRequest{ + let cor = CopyObjectRequest { bucket: self.bucket.clone(), key: format!("{}/{}", self.mail_path, new_id), copy_source: format!("{}/{}/{}", from.bucket, from.mail_path, source_id), diff --git a/src/mail/user.rs b/src/mail/user.rs index 44e0081..5523c2a 100644 --- a/src/mail/user.rs +++ b/src/mail/user.rs @@ -334,17 +334,22 @@ impl MailboxList { } fn has_mailbox(&self, name: &str) -> bool { - matches!(self.0.get(name), Some(MailboxListEntry { - id_lww: (_, Some(_)), - .. - })) + matches!( + self.0.get(name), + Some(MailboxListEntry { + id_lww: (_, Some(_)), + .. + }) + ) } fn get_mailbox(&self, name: &str) -> Option<(ImapUidvalidity, Option)> { - self.0.get(name).map(|MailboxListEntry { - id_lww: (_, mailbox_id), - uidvalidity, - }| (*uidvalidity, *mailbox_id)) + self.0.get(name).map( + |MailboxListEntry { + id_lww: (_, mailbox_id), + uidvalidity, + }| (*uidvalidity, *mailbox_id), + ) } /// Ensures mailbox `name` maps to id `id`.