Deuxfleurs/aerogramme
15
0
Fork 1
Code Issues 33 Pull requests 1 Projects 11 Releases 4 Wiki Activity

basic body ext testing + format

Browse source
This commit is contained in:
Quentin 2023-10-12 12:21:59 +02:00
parent b444ef7ef3
commit a1b7ca17c0
Signed by: quentin
GPG key ID: E9602264D639FF68
6 changed files with 379 additions and 162 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/bayou.rs
View file

@ -384,7 +384,7 @@ impl<S: BayouState> Bayou<S> {
let cryptoblob = seal_serialize(&state_cp, &self.key)?; let cryptoblob = seal_serialize(&state_cp, &self.key)?;
debug!("(cp) checkpoint body length: {}", cryptoblob.len()); debug!("(cp) checkpoint body length: {}", cryptoblob.len());
let por = PutObjectRequest{ let por = PutObjectRequest {
bucket: self.bucket.clone(), bucket: self.bucket.clone(),
key: format!("{}/checkpoint/{}", self.path, ts_cp.to_string()), key: format!("{}/checkpoint/{}", self.path, ts_cp.to_string()),
body: Some(cryptoblob.into()), body: Some(cryptoblob.into()),
@ -437,7 +437,7 @@ impl<S: BayouState> Bayou<S> {
async fn list_checkpoints(&self) -> Result<Vec<(Timestamp, String)>> { async fn list_checkpoints(&self) -> Result<Vec<(Timestamp, String)>> {
let prefix = format!("{}/checkpoint/", self.path); let prefix = format!("{}/checkpoint/", self.path);
let lor = ListObjectsV2Request{ let lor = ListObjectsV2Request {
bucket: self.bucket.clone(), bucket: self.bucket.clone(),
max_keys: Some(1000), max_keys: Some(1000),
prefix: Some(prefix.clone()), prefix: Some(prefix.clone()),

486
src/imap/mailbox_view.rs
View file

@ -1,7 +1,7 @@
use std::borrow::Cow; use std::borrow::Cow;
use std::iter::zip;
use std::num::NonZeroU32; use std::num::NonZeroU32;
use std::sync::Arc; use std::sync::Arc;
use std::iter::zip;
use anyhow::{anyhow, bail, Error, Result}; use anyhow::{anyhow, bail, Error, Result};
use boitalettres::proto::res::body::Data as Body; use boitalettres::proto::res::body::Data as Body;
@ -22,15 +22,13 @@ use imap_codec::types::response::{Code, Data, MessageAttribute, Status};
use imap_codec::types::sequence::{self, SequenceSet}; use imap_codec::types::sequence::{self, SequenceSet};
use eml_codec::{ use eml_codec::{
header, header, imf, mime,
imf,
part::{AnyPart, composite::Message},
mime::r#type::Deductible, mime::r#type::Deductible,
mime, part::{composite::Message, AnyPart},
}; };
use crate::cryptoblob::Key; use crate::cryptoblob::Key;
use crate::mail::mailbox::{Mailbox, MailMeta}; use crate::mail::mailbox::{MailMeta, Mailbox};
use crate::mail::uidindex::{ImapUid, ImapUidvalidity, UidIndex}; use crate::mail::uidindex::{ImapUid, ImapUidvalidity, UidIndex};
use crate::mail::unique_ident::UniqueIdent; use crate::mail::unique_ident::UniqueIdent;
@ -44,26 +42,42 @@ const DEFAULT_FLAGS: [Flag; 5] = [
enum FetchedMail<'a> { enum FetchedMail<'a> {
Partial(imf::Imf<'a>), Partial(imf::Imf<'a>),
Full(Message<'a>), Full(AnyPart<'a>),
} }
impl<'a> FetchedMail<'a> { impl<'a> FetchedMail<'a> {
fn as_full(&self) -> Result<&Message<'a>> { fn new_from_message(msg: Message<'a>) -> Self {
FetchedMail::Full(AnyPart::Msg(msg))
}
/*fn new_from_header(hdr: imf::Imf<'a>) -> Self {
FetchedMail::Partial(hdr)
}*/
fn as_anypart(&self) -> Result<&AnyPart<'a>> {
match self { match self {
FetchedMail::Full(x) => Ok(&x), FetchedMail::Full(x) => Ok(&x),
_ => bail!("The full message must be fetched, not only its headers: it's a logic error"), _ => bail!("The full message must be fetched, not only its headers"),
}
}
fn as_full(&self) -> Result<&Message<'a>> {
match self {
FetchedMail::Full(AnyPart::Msg(x)) => Ok(&x),
_ => bail!("The full message must be fetched, not only its headers AND it must be an AnyPart::Msg."),
} }
} }
fn imf(&self) -> &imf::Imf<'a> { fn imf(&self) -> &imf::Imf<'a> {
match self { match self {
FetchedMail::Full(x) => &x.imf, FetchedMail::Full(AnyPart::Msg(x)) => &x.imf,
FetchedMail::Partial(x) => &x, FetchedMail::Partial(x) => &x,
_ => panic!("Can't contain AnyPart that is not a message"),
} }
} }
} }
pub struct AttributesProxy { pub struct AttributesProxy {
attrs: Vec<FetchAttribute> attrs: Vec<FetchAttribute>,
} }
impl AttributesProxy { impl AttributesProxy {
fn new(attrs: &MacroOrFetchAttributes, is_uid_fetch: bool) -> Self { fn new(attrs: &MacroOrFetchAttributes, is_uid_fetch: bool) -> Self {
@ -112,8 +126,8 @@ pub struct MailView<'a> {
ids: &'a MailIdentifiers, ids: &'a MailIdentifiers,
meta: &'a MailMeta, meta: &'a MailMeta,
flags: &'a Vec<String>, flags: &'a Vec<String>,
content: FetchedMail<'a>, content: FetchedMail<'a>,
add_seen: bool add_seen: bool,
} }
impl<'a> MailView<'a> { impl<'a> MailView<'a> {
@ -123,7 +137,10 @@ impl<'a> MailView<'a> {
fn flags(&self) -> MessageAttribute { fn flags(&self) -> MessageAttribute {
MessageAttribute::Flags( MessageAttribute::Flags(
self.flags.iter().filter_map(|f| string_to_flag(f)).collect(), self.flags
.iter()
.filter_map(|f| string_to_flag(f))
.collect(),
) )
} }
@ -132,18 +149,37 @@ impl<'a> MailView<'a> {
} }
fn rfc_822_header(&self) -> MessageAttribute { fn rfc_822_header(&self) -> MessageAttribute {
MessageAttribute::Rfc822Header(NString(self.meta.headers.to_vec().try_into().ok().map(IString::Literal))) MessageAttribute::Rfc822Header(NString(
self.meta
.headers
.to_vec()
.try_into()
.ok()
.map(IString::Literal),
))
} }
fn rfc_822_text(&self) -> Result<MessageAttribute> { fn rfc_822_text(&self) -> Result<MessageAttribute> {
Ok(MessageAttribute::Rfc822Text(NString( Ok(MessageAttribute::Rfc822Text(NString(
self.content.as_full()?.raw_body.try_into().ok().map(IString::Literal), self.content
.as_full()?
.raw_body
.try_into()
.ok()
.map(IString::Literal),
))) )))
} }
fn rfc822(&self) -> Result<MessageAttribute> { fn rfc822(&self) -> Result<MessageAttribute> {
Ok(MessageAttribute::Rfc822(NString( Ok(MessageAttribute::Rfc822(NString(
self.content.as_full()?.raw_body.clone().try_into().ok().map(IString::Literal)))) self.content
.as_full()?
.raw_body
.clone()
.try_into()
.ok()
.map(IString::Literal),
)))
} }
fn envelope(&self) -> MessageAttribute { fn envelope(&self) -> MessageAttribute {
@ -151,24 +187,29 @@ impl<'a> MailView<'a> {
} }
fn body(&self) -> Result<MessageAttribute> { fn body(&self) -> Result<MessageAttribute> {
Ok(MessageAttribute::Body( Ok(MessageAttribute::Body(build_imap_email_struct(
build_imap_email_struct(self.content.as_full()?.child.as_ref())?, self.content.as_full()?.child.as_ref(),
)) )?))
} }
fn body_structure(&self) -> Result<MessageAttribute> { fn body_structure(&self) -> Result<MessageAttribute> {
Ok(MessageAttribute::Body( Ok(MessageAttribute::Body(build_imap_email_struct(
build_imap_email_struct(self.content.as_full()?.child.as_ref())?, self.content.as_full()?.child.as_ref(),
)) )?))
} }
/// maps to BODY[<section>]<<partial>> and BODY.PEEK[<section>]<<partial>> /// maps to BODY[<section>]<<partial>> and BODY.PEEK[<section>]<<partial>>
/// peek does not implicitly set the \Seen flag /// peek does not implicitly set the \Seen flag
/// eg. BODY[HEADER.FIELDS (DATE FROM)] /// eg. BODY[HEADER.FIELDS (DATE FROM)]
/// eg. BODY[]<0.2048> /// eg. BODY[]<0.2048>
fn body_ext(&mut self, section: &Option<FetchSection>, partial: &Option<(u32, NonZeroU32)>, peek: &bool) -> Result<MessageAttribute> { fn body_ext(
&mut self,
section: &Option<FetchSection>,
partial: &Option<(u32, NonZeroU32)>,
peek: &bool,
) -> Result<MessageAttribute> {
// Extract message section // Extract message section
let text = get_message_section(self.content.as_full()?, section)?; let text = get_message_section(self.content.as_anypart()?, section)?;
let seen_flag = Flag::Seen.to_string(); let seen_flag = Flag::Seen.to_string();
if !peek && !self.flags.iter().any(|x| *x == seen_flag) { if !peek && !self.flags.iter().any(|x| *x == seen_flag) {
@ -186,29 +227,40 @@ impl<'a> MailView<'a> {
section: section.clone(), section: section.clone(),
origin, origin,
data, data,
}) });
} }
fn internal_date(&self) -> Result<MessageAttribute> { fn internal_date(&self) -> Result<MessageAttribute> {
let dt = Utc.fix().timestamp_opt(i64::try_from(self.meta.internaldate / 1000)?, 0).earliest().ok_or(anyhow!("Unable to parse internal date"))?; let dt = Utc
.fix()
.timestamp_opt(i64::try_from(self.meta.internaldate / 1000)?, 0)
.earliest()
.ok_or(anyhow!("Unable to parse internal date"))?;
Ok(MessageAttribute::InternalDate(MyDateTime(dt))) Ok(MessageAttribute::InternalDate(MyDateTime(dt)))
} }
fn filter(&mut self, ap: &AttributesProxy) -> Result<Body> { fn filter(&mut self, ap: &AttributesProxy) -> Result<Body> {
let res_attrs = ap.attrs.iter().map(|attr| match attr { let res_attrs = ap
FetchAttribute::Uid => Ok(self.uid()), .attrs
FetchAttribute::Flags => Ok(self.flags()), .iter()
FetchAttribute::Rfc822Size => Ok(self.rfc_822_size()), .map(|attr| match attr {
FetchAttribute::Rfc822Header => Ok(self.rfc_822_header()), FetchAttribute::Uid => Ok(self.uid()),
FetchAttribute::Rfc822Text => self.rfc_822_text(), FetchAttribute::Flags => Ok(self.flags()),
FetchAttribute::Rfc822 => self.rfc822(), FetchAttribute::Rfc822Size => Ok(self.rfc_822_size()),
FetchAttribute::Envelope => Ok(self.envelope()), FetchAttribute::Rfc822Header => Ok(self.rfc_822_header()),
FetchAttribute::Body => self.body(), FetchAttribute::Rfc822Text => self.rfc_822_text(),
FetchAttribute::BodyStructure => self.body_structure(), FetchAttribute::Rfc822 => self.rfc822(),
FetchAttribute::BodyExt { section, partial, peek } => self.body_ext(section, partial, peek), FetchAttribute::Envelope => Ok(self.envelope()),
FetchAttribute::InternalDate => self.internal_date(), FetchAttribute::Body => self.body(),
}).collect::<Result<Vec<_>, _>>()?; FetchAttribute::BodyStructure => self.body_structure(),
FetchAttribute::BodyExt {
section,
partial,
peek,
} => self.body_ext(section, partial, peek),
FetchAttribute::InternalDate => self.internal_date(),
})
.collect::<Result<Vec<_>, _>>()?;
Ok(Body::Data(Data::Fetch { Ok(Body::Data(Data::Fetch {
seq_or_uid: self.ids.i, seq_or_uid: self.ids.i,
@ -217,7 +269,10 @@ impl<'a> MailView<'a> {
} }
} }
fn apply_partial<'a>(partial: &'_ Option<(u32, NonZeroU32)>, text: &'a [u8]) -> (&'a [u8], Option<u32>) { fn apply_partial<'a>(
partial: &'_ Option<(u32, NonZeroU32)>,
text: &'a [u8],
) -> (&'a [u8], Option<u32>) {
match partial { match partial {
Some((begin, len)) => { Some((begin, len)) => {
if *begin as usize > text.len() { if *begin as usize > text.len() {
@ -253,11 +308,11 @@ pub struct MailSelectionBuilder<'a> {
impl<'a> MailSelectionBuilder<'a> { impl<'a> MailSelectionBuilder<'a> {
fn new(need_body: bool, mail_count: usize) -> Self { fn new(need_body: bool, mail_count: usize) -> Self {
Self { Self {
mail_count, mail_count,
need_body, need_body,
..MailSelectionBuilder::default() ..MailSelectionBuilder::default()
} }
} }
fn with_mail_identifiers(&mut self, mi: &'a [MailIdentifiers]) -> &mut Self { fn with_mail_identifiers(&mut self, mi: &'a [MailIdentifiers]) -> &mut Self {
@ -269,7 +324,7 @@ impl<'a> MailSelectionBuilder<'a> {
self.meta = meta; self.meta = meta;
self self
} }
fn with_flags(&mut self, flags: &'a [&'a Vec<String>]) -> &mut Self { fn with_flags(&mut self, flags: &'a [&'a Vec<String>]) -> &mut Self {
self.flags = flags; self.flags = flags;
self self
@ -277,9 +332,14 @@ impl<'a> MailSelectionBuilder<'a> {
fn bodies_to_collect(&self) -> Vec<BodyIdentifier> { fn bodies_to_collect(&self) -> Vec<BodyIdentifier> {
if !self.need_body { if !self.need_body {
return vec![] return vec![];
} }
zip(self.mi, self.meta).map(|(mi, meta)| BodyIdentifier { msg_uuid: &mi.uuid, msg_key: &meta.message_key }).collect::<Vec<_>>() zip(self.mi, self.meta)
.map(|(mi, meta)| BodyIdentifier {
msg_uuid: &mi.uuid,
msg_key: &meta.message_key,
})
.collect::<Vec<_>>()
} }
fn with_bodies(&mut self, rbodies: &'a [Vec<u8>]) -> &mut Self { fn with_bodies(&mut self, rbodies: &'a [Vec<u8>]) -> &mut Self {
@ -292,22 +352,32 @@ impl<'a> MailSelectionBuilder<'a> {
if !self.need_body { if !self.need_body {
for m in self.meta.iter() { for m in self.meta.iter() {
let (_, hdrs) = eml_codec::parse_imf(&m.headers).or(Err(anyhow!("Invalid mail headers")))?; let (_, hdrs) =
eml_codec::parse_imf(&m.headers).or(Err(anyhow!("Invalid mail headers")))?;
bodies.push(FetchedMail::Partial(hdrs)); bodies.push(FetchedMail::Partial(hdrs));
} }
} else { } else {
for rb in self.bodies.iter() { for rb in self.bodies.iter() {
let (_, p) = eml_codec::parse_message(&rb).or(Err(anyhow!("Invalid mail body")))?; let (_, p) = eml_codec::parse_message(&rb).or(Err(anyhow!("Invalid mail body")))?;
bodies.push(FetchedMail::Full(p)); bodies.push(FetchedMail::new_from_message(p));
} }
} }
if self.mi.len() != self.mail_count && self.meta.len() != self.mail_count || self.flags.len() != self.mail_count || bodies.len() != self.mail_count { if self.mi.len() != self.mail_count && self.meta.len() != self.mail_count
return Err(anyhow!("Can't build a mail view selection as parts were not correctly registered into the builder.")) || self.flags.len() != self.mail_count
|| bodies.len() != self.mail_count
{
return Err(anyhow!("Can't build a mail view selection as parts were not correctly registered into the builder."));
} }
Ok(zip(self.mi, zip(self.meta, zip(self.flags, bodies))) Ok(zip(self.mi, zip(self.meta, zip(self.flags, bodies)))
.map(|(ids, (meta, (flags, content)))| MailView { ids, meta, flags, content, add_seen: false }) .map(|(ids, (meta, (flags, content)))| MailView {
ids,
meta,
flags,
content,
add_seen: false,
})
.collect()) .collect())
} }
} }
@ -505,7 +575,6 @@ impl MailboxView {
attributes: &MacroOrFetchAttributes, attributes: &MacroOrFetchAttributes,
is_uid_fetch: &bool, is_uid_fetch: &bool,
) -> Result<Vec<Body>> { ) -> Result<Vec<Body>> {
let ap = AttributesProxy::new(attributes, *is_uid_fetch); let ap = AttributesProxy::new(attributes, *is_uid_fetch);
// Prepare data // Prepare data
@ -513,22 +582,38 @@ impl MailboxView {
let mail_count = mids.0.len(); let mail_count = mids.0.len();
let uuids = mids.uuids(); let uuids = mids.uuids();
let meta = self.mailbox.fetch_meta(&uuids).await?; let meta = self.mailbox.fetch_meta(&uuids).await?;
let flags = uuids.iter().map(|uuid| self.known_state.table.get(uuid).map(|(_uuid, f)| f).ok_or(anyhow!("missing email from the flag table"))).collect::<Result<Vec<_>, _>>()?; let flags = uuids
.iter()
.map(|uuid| {
self.known_state
.table
.get(uuid)
.map(|(_uuid, f)| f)
.ok_or(anyhow!("missing email from the flag table"))
})
.collect::<Result<Vec<_>, _>>()?;
// Start filling data to build the view // Start filling data to build the view
let mut selection = MailSelectionBuilder::new(ap.need_body(), mail_count); let mut selection = MailSelectionBuilder::new(ap.need_body(), mail_count);
selection selection
.with_mail_identifiers(&mids.0) .with_mail_identifiers(&mids.0)
.with_metadata(&meta) .with_metadata(&meta)
.with_flags(&flags); .with_flags(&flags);
// Asynchronously fetch full bodies (if needed) // Asynchronously fetch full bodies (if needed)
let btc = selection.bodies_to_collect(); let btc = selection.bodies_to_collect();
let future_bodies = btc.iter().map(|bi| async move { let future_bodies = btc
let body = self.mailbox.fetch_full(*bi.msg_uuid, bi.msg_key).await?; .iter()
Ok::<_, anyhow::Error>(body) .map(|bi| async move {
}).collect::<FuturesOrdered<_>>(); let body = self.mailbox.fetch_full(*bi.msg_uuid, bi.msg_key).await?;
let bodies = future_bodies.collect::<Vec<_>>().await.into_iter().collect::<Result<Vec<_>, _>>()?; Ok::<_, anyhow::Error>(body)
})
.collect::<FuturesOrdered<_>>();
let bodies = future_bodies
.collect::<Vec<_>>()
.await
.into_iter()
.collect::<Result<Vec<_>, _>>()?;
// Add bodies // Add bodies
selection.with_bodies(bodies.as_slice()); selection.with_bodies(bodies.as_slice());
@ -537,15 +622,26 @@ impl MailboxView {
let mut views = selection.build()?; let mut views = selection.build()?;
// Filter views to build the result // Filter views to build the result
let ret = views.iter_mut().filter_map(|mv| mv.filter(&ap).ok()).collect::<Vec<_>>(); let ret = views
.iter_mut()
.filter_map(|mv| mv.filter(&ap).ok())
.collect::<Vec<_>>();
// Register seen flags // Register seen flags
let future_flags = views.iter().filter(|mv| mv.add_seen).map(|mv| async move { let future_flags = views
let seen_flag = Flag::Seen.to_string(); .iter()
self.mailbox.add_flags(mv.ids.uuid, &[seen_flag]).await?; .filter(|mv| mv.add_seen)
Ok::<_, anyhow::Error>(()) .map(|mv| async move {
}).collect::<FuturesOrdered<_>>(); let seen_flag = Flag::Seen.to_string();
future_flags.collect::<Vec<_>>().await.into_iter().collect::<Result<_, _>>()?; self.mailbox.add_flags(mv.ids.uuid, &[seen_flag]).await?;
Ok::<_, anyhow::Error>(())
})
.collect::<FuturesOrdered<_>>();
future_flags
.collect::<Vec<_>>()
.await
.into_iter()
.collect::<Result<_, _>>()?;
Ok(ret) Ok(ret)
} }
@ -583,7 +679,11 @@ impl MailboxView {
} }
if let Some(mail) = mail_vec.get(i) { if let Some(mail) = mail_vec.get(i) {
if mail.0 == uid { if mail.0 == uid {
mails.push(MailIdentifiers { i: NonZeroU32::try_from(i as u32 + 1).unwrap(), uid: mail.0, uuid: mail.1 }); mails.push(MailIdentifiers {
i: NonZeroU32::try_from(i as u32 + 1).unwrap(),
uid: mail.0,
uuid: mail.1,
});
} }
} else { } else {
break; break;
@ -600,7 +700,11 @@ impl MailboxView {
for i in sequence_set.iter(iter_strat) { for i in sequence_set.iter(iter_strat) {
if let Some(mail) = mail_vec.get(i.get() as usize - 1) { if let Some(mail) = mail_vec.get(i.get() as usize - 1) {
mails.push(MailIdentifiers { i, uid: mail.0, uuid: mail.1 }); mails.push(MailIdentifiers {
i,
uid: mail.0,
uuid: mail.1,
});
} else { } else {
bail!("No such mail: {}", i); bail!("No such mail: {}", i);
} }
@ -759,26 +863,39 @@ fn message_envelope(msg: &imf::Imf) -> Envelope {
Envelope { Envelope {
date: NString( date: NString(
msg.date.as_ref() msg.date
.as_ref()
.map(|d| IString::try_from(d.to_rfc3339()).unwrap()), .map(|d| IString::try_from(d.to_rfc3339()).unwrap()),
), ),
subject: NString( subject: NString(
msg.subject.as_ref() msg.subject
.as_ref()
.map(|d| IString::try_from(d.to_string()).unwrap()), .map(|d| IString::try_from(d.to_string()).unwrap()),
), ),
sender: msg.sender.as_ref().map(|v| vec![convert_mbx(v)]).unwrap_or(from.clone()), sender: msg
.sender
.as_ref()
.map(|v| vec![convert_mbx(v)])
.unwrap_or(from.clone()),
reply_to: if msg.reply_to.is_empty() { reply_to: if msg.reply_to.is_empty() {
from.clone() from.clone()
} else { } else {
convert_addresses(&msg.reply_to) convert_addresses(&msg.reply_to)
}, },
from, from,
to: convert_addresses(&msg.to), to: convert_addresses(&msg.to),
cc: convert_addresses(&msg.cc), cc: convert_addresses(&msg.cc),
bcc: convert_addresses(&msg.bcc), bcc: convert_addresses(&msg.bcc),
in_reply_to: NString(msg.in_reply_to.iter().next().map(|d| IString::try_from(d.to_string()).unwrap())), in_reply_to: NString(
msg.in_reply_to
.iter()
.next()
.map(|d| IString::try_from(d.to_string()).unwrap()),
),
message_id: NString( message_id: NString(
msg.msg_id.as_ref().map(|d| IString::try_from(d.to_string()).unwrap()), msg.msg_id
.as_ref()
.map(|d| IString::try_from(d.to_string()).unwrap()),
), ),
} }
} }
@ -788,20 +905,28 @@ fn convert_addresses(addrlist: &Vec<imf::address::AddressRef>) -> Vec<Address> {
for item in addrlist { for item in addrlist {
match item { match item {
imf::address::AddressRef::Single(a) => acc.push(convert_mbx(a)), imf::address::AddressRef::Single(a) => acc.push(convert_mbx(a)),
imf::address::AddressRef::Many(l) => acc.extend(l.participants.iter().map(convert_mbx)) imf::address::AddressRef::Many(l) => acc.extend(l.participants.iter().map(convert_mbx)),
} }
} }
return acc return acc;
} }
fn convert_mbx(addr: &imf::mailbox::MailboxRef) -> Address { fn convert_mbx(addr: &imf::mailbox::MailboxRef) -> Address {
Address::new( Address::new(
NString(addr.name.as_ref().map(|x| IString::try_from(x.to_string()).unwrap())), NString(
addr.name
.as_ref()
.map(|x| IString::try_from(x.to_string()).unwrap()),
),
// SMTP at-domain-list (source route) seems obsolete since at least 1991 // SMTP at-domain-list (source route) seems obsolete since at least 1991
// https://www.mhonarc.org/archive/html/ietf-822/1991-06/msg00060.html // https://www.mhonarc.org/archive/html/ietf-822/1991-06/msg00060.html
NString(None), NString(None),
NString(Some(IString::try_from(addr.addrspec.local_part.to_string()).unwrap())), NString(Some(
NString(Some(IString::try_from(addr.addrspec.domain.to_string()).unwrap())), IString::try_from(addr.addrspec.local_part.to_string()).unwrap(),
)),
NString(Some(
IString::try_from(addr.addrspec.domain.to_string()).unwrap(),
)),
) )
} }
@ -824,10 +949,12 @@ fn build_imap_email_struct<'a>(part: &AnyPart<'a>) -> Result<BodyStructure> {
match part { match part {
AnyPart::Mult(x) => { AnyPart::Mult(x) => {
let itype = &x.mime.interpreted_type; let itype = &x.mime.interpreted_type;
let subtype = IString::try_from(itype.subtype.to_string()).unwrap_or(unchecked_istring("alternative")); let subtype = IString::try_from(itype.subtype.to_string())
.unwrap_or(unchecked_istring("alternative"));
Ok(BodyStructure::Multi { Ok(BodyStructure::Multi {
bodies: x.children bodies: x
.children
.iter() .iter()
.filter_map(|inner| build_imap_email_struct(&inner).ok()) .filter_map(|inner| build_imap_email_struct(&inner).ok())
.collect(), .collect(),
@ -845,18 +972,19 @@ fn build_imap_email_struct<'a>(part: &AnyPart<'a>) -> Result<BodyStructure> {
AnyPart::Txt(x) => { AnyPart::Txt(x) => {
let mut basic = basic_fields(&x.mime.fields, x.body.len())?; let mut basic = basic_fields(&x.mime.fields, x.body.len())?;
// Get the interpreted content type, set it // Get the interpreted content type, set it
let itype = match &x.mime.interpreted_type { let itype = match &x.mime.interpreted_type {
Deductible::Inferred(v) | Deductible::Explicit(v) => v Deductible::Inferred(v) | Deductible::Explicit(v) => v,
}; };
let subtype = IString::try_from(itype.subtype.to_string()).unwrap_or(unchecked_istring("plain")); let subtype =
IString::try_from(itype.subtype.to_string()).unwrap_or(unchecked_istring("plain"));
// Add charset to the list of parameters if we know it has been inferred as it will be // Add charset to the list of parameters if we know it has been inferred as it will be
// missing from the parsed content. // missing from the parsed content.
if let Deductible::Inferred(charset) = &itype.charset { if let Deductible::Inferred(charset) = &itype.charset {
basic.parameter_list.push(( basic.parameter_list.push((
unchecked_istring("charset"), unchecked_istring("charset"),
IString::try_from(charset.to_string()).unwrap_or(unchecked_istring("us-ascii")) IString::try_from(charset.to_string()).unwrap_or(unchecked_istring("us-ascii")),
)); ));
} }
@ -874,15 +1002,21 @@ fn build_imap_email_struct<'a>(part: &AnyPart<'a>) -> Result<BodyStructure> {
AnyPart::Bin(x) => { AnyPart::Bin(x) => {
let basic = basic_fields(&x.mime.fields, x.body.len())?; let basic = basic_fields(&x.mime.fields, x.body.len())?;
let default = mime::r#type::NaiveType { main: &b"application"[..], sub: &b"octet-stream"[..], params: vec![] }; let default = mime::r#type::NaiveType {
main: &b"application"[..],
sub: &b"octet-stream"[..],
params: vec![],
};
let ct = x.mime.fields.ctype.as_ref().unwrap_or(&default); let ct = x.mime.fields.ctype.as_ref().unwrap_or(&default);
let type_ = IString::try_from(String::from_utf8_lossy(ct.main).to_string()) let type_ = IString::try_from(String::from_utf8_lossy(ct.main).to_string()).or(Err(
.or(Err(anyhow!("Unable to build IString from given Content-Type type given")))?; anyhow!("Unable to build IString from given Content-Type type given"),
))?;
let subtype =
let subtype = IString::try_from(String::from_utf8_lossy(ct.sub).to_string()) IString::try_from(String::from_utf8_lossy(ct.sub).to_string()).or(Err(anyhow!(
.or(Err(anyhow!("Unable to build IString from given Content-Type subtype given")))?; "Unable to build IString from given Content-Type subtype given"
)))?;
Ok(BodyStructure::Single { Ok(BodyStructure::Single {
body: FetchBody { body: FetchBody {
@ -911,7 +1045,8 @@ fn build_imap_email_struct<'a>(part: &AnyPart<'a>) -> Result<BodyStructure> {
} }
fn nol(input: &[u8]) -> u32 { fn nol(input: &[u8]) -> u32 {
input.iter() input
.iter()
.filter(|x| **x == b'\n') .filter(|x| **x == b'\n')
.count() .count()
.try_into() .try_into()
@ -925,13 +1060,22 @@ fn unchecked_istring(s: &'static str) -> IString {
} }
fn basic_fields(m: &mime::NaiveMIME, sz: usize) -> Result<BasicFields> { fn basic_fields(m: &mime::NaiveMIME, sz: usize) -> Result<BasicFields> {
let parameter_list = m.ctype let parameter_list = m
.ctype
.as_ref() .as_ref()
.map(|x| x.params.iter() .map(|x| {
.map(|p| (IString::try_from(String::from_utf8_lossy(p.name).to_string()), IString::try_from(p.value.to_string()))) x.params
.filter(|(k, v)| k.is_ok() && v.is_ok()) .iter()
.map(|(k, v)| (k.unwrap(), v.unwrap())) .map(|p| {
.collect()) (
IString::try_from(String::from_utf8_lossy(p.name).to_string()),
IString::try_from(p.value.to_string()),
)
})
.filter(|(k, v)| k.is_ok() && v.is_ok())
.map(|(k, v)| (k.unwrap(), v.unwrap()))
.collect()
})
.unwrap_or(vec![]); .unwrap_or(vec![]);
Ok(BasicFields { Ok(BasicFields {
@ -939,17 +1083,18 @@ fn basic_fields(m: &mime::NaiveMIME, sz: usize) -> Result<BasicFields> {
id: NString( id: NString(
m.id.as_ref() m.id.as_ref()
.and_then(|ci| IString::try_from(ci.to_string()).ok()), .and_then(|ci| IString::try_from(ci.to_string()).ok()),
), ),
description: NString( description: NString(
m.description.as_ref() m.description
.as_ref()
.and_then(|cd| IString::try_from(cd.to_string()).ok()), .and_then(|cd| IString::try_from(cd.to_string()).ok()),
), ),
content_transfer_encoding: match m.transfer_encoding { content_transfer_encoding: match m.transfer_encoding {
mime::mechanism::Mechanism::_8Bit => unchecked_istring("8bit"), mime::mechanism::Mechanism::_8Bit => unchecked_istring("8bit"),
mime::mechanism::Mechanism::Binary => unchecked_istring("binary"), mime::mechanism::Mechanism::Binary => unchecked_istring("binary"),
mime::mechanism::Mechanism::QuotedPrintable => unchecked_istring("quoted-printable"), mime::mechanism::Mechanism::QuotedPrintable => unchecked_istring("quoted-printable"),
mime::mechanism::Mechanism::Base64 => unchecked_istring("base64"), mime::mechanism::Mechanism::Base64 => unchecked_istring("base64"),
_ => unchecked_istring("7bit"), _ => unchecked_istring("7bit"),
}, },
// @FIXME we can't compute the size of the message currently... // @FIXME we can't compute the size of the message currently...
size: u32::try_from(sz)?, size: u32::try_from(sz)?,
@ -983,25 +1128,34 @@ fn basic_fields(m: &mime::NaiveMIME, sz: usize) -> Result<BasicFields> {
/// 4.2.2.2 TEXT/RICHTEXT /// 4.2.2.2 TEXT/RICHTEXT
/// ``` /// ```
fn get_message_section<'a>( fn get_message_section<'a>(
parsed: &'a Message<'a>, parsed: &'a AnyPart<'a>,
section: &Option<FetchSection>, section: &Option<FetchSection>,
) -> Result<Cow<'a, [u8]>> { ) -> Result<Cow<'a, [u8]>> {
let msg = parsed
.as_message()
.ok_or(anyhow!("Part must be a message"))?;
match section { match section {
Some(FetchSection::Text(None)) => { Some(FetchSection::Text(None)) => Ok(msg.raw_body.into()),
Ok(parsed.raw_body.into())
}
Some(FetchSection::Text(Some(part))) => { Some(FetchSection::Text(Some(part))) => {
map_subpart(parsed.child.as_ref(), part.0.as_slice(), |part_msg| { map_subpart(parsed, part.0.as_slice(), |part_msg| {
Ok(part_msg.as_message().ok_or(Error::msg("Not a message/rfc822 part while expected by request (xxx.TEXT)"))? Ok(part_msg
.as_message()
.ok_or(Error::msg(
"Not a message/rfc822 part while expected by request (TEXT)",
))?
.raw_body .raw_body
.into()) .into())
}) })
} }
Some(FetchSection::Header(part)) => map_subpart( Some(FetchSection::Header(part)) => map_subpart(
parsed.child.as_ref(), parsed,
part.as_ref().map(|p| p.0.as_slice()).unwrap_or(&[]), part.as_ref().map(|p| p.0.as_slice()).unwrap_or(&[]),
|part_msg| { |part_msg| {
Ok(part_msg.as_message().ok_or(Error::msg("Not a message/rfc822 part while expected by request (xxx.TEXT)"))? Ok(part_msg
.as_message()
.ok_or(Error::msg(
"Not a message/rfc822 part while expected by request (HEADER)",
))?
.raw_headers .raw_headers
.into()) .into())
}, },
@ -1019,9 +1173,8 @@ fn get_message_section<'a>(
}) })
.collect::<Vec<_>>(); .collect::<Vec<_>>();
map_subpart( map_subpart(
parsed.child.as_ref(), parsed,
part.as_ref().map(|p| p.0.as_slice()).unwrap_or(&[]), part.as_ref().map(|p| p.0.as_slice()).unwrap_or(&[]),
|part_msg| { |part_msg| {
let mut ret = vec![]; let mut ret = vec![];
@ -1030,11 +1183,7 @@ fn get_message_section<'a>(
header::Field::Good(header::Kv2(k, v)) => (k, v), header::Field::Good(header::Kv2(k, v)) => (k, v),
_ => continue, _ => continue,
}; };
if fields if fields.as_slice().iter().any(|x| (x == k) ^ invert) {
.as_slice()
.iter()
.any(|x| (x == k) ^ invert)
{
ret.extend(*k); ret.extend(*k);
ret.extend(b": "); ret.extend(b": ");
ret.extend(*v); ret.extend(*v);
@ -1046,7 +1195,7 @@ fn get_message_section<'a>(
}, },
) )
} }
Some(FetchSection::Part(part)) => map_subpart(parsed.child.as_ref(), part.0.as_slice(), |part| { Some(FetchSection::Part(part)) => map_subpart(parsed, part.0.as_slice(), |part| {
let bytes = match &part { let bytes = match &part {
AnyPart::Txt(p) => p.body, AnyPart::Txt(p) => p.body,
AnyPart::Bin(p) => p.body, AnyPart::Bin(p) => p.body,
@ -1055,7 +1204,7 @@ fn get_message_section<'a>(
}; };
Ok(bytes.to_vec().into()) Ok(bytes.to_vec().into())
}), }),
Some(FetchSection::Mime(part)) => map_subpart(parsed.child.as_ref(), part.0.as_slice(), |part| { Some(FetchSection::Mime(part)) => map_subpart(parsed, part.0.as_slice(), |part| {
let bytes = match &part { let bytes = match &part {
AnyPart::Txt(p) => p.mime.fields.raw, AnyPart::Txt(p) => p.mime.fields.raw,
AnyPart::Bin(p) => p.mime.fields.raw, AnyPart::Bin(p) => p.mime.fields.raw,
@ -1064,11 +1213,11 @@ fn get_message_section<'a>(
}; };
Ok(bytes.to_vec().into()) Ok(bytes.to_vec().into())
}), }),
None => Ok(parsed.raw_part.into()), None => Ok(msg.raw_part.into()),
} }
} }
/// Fetch a MIME SubPart /// Fetch a MIME SubPart
/// ///
/// eg. FETCH BODY[4.2.2.1] -> [4, 2, 2, 1] /// eg. FETCH BODY[4.2.2.1] -> [4, 2, 2, 1]
fn map_subpart<'a, F, R>(part: &AnyPart<'a>, path: &[NonZeroU32], f: F) -> Result<R> fn map_subpart<'a, F, R>(part: &AnyPart<'a>, path: &[NonZeroU32], f: F) -> Result<R>
@ -1094,9 +1243,72 @@ where
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use crate::cryptoblob;
use crate::mail::unique_ident;
use imap_codec::codec::Encode; use imap_codec::codec::Encode;
use imap_codec::types::fetch_attributes::Section;
use std::fs; use std::fs;
#[test]
fn mailview_body_ext() -> Result<()> {
let ap = AttributesProxy::new(
&MacroOrFetchAttributes::FetchAttributes(vec![FetchAttribute::BodyExt {
section: Some(Section::Header(None)),
partial: None,
peek: false,
}]),
false,
);
let flags = vec![];
let key = cryptoblob::gen_key();
let meta = MailMeta {
internaldate: 0u64,
headers: vec![],
message_key: key,
rfc822_size: 8usize,
};
let ids = MailIdentifiers {
i: NonZeroU32::MIN,
uid: NonZeroU32::MIN,
uuid: unique_ident::gen_ident(),
};
let rfc822 = b"Subject: hello\r\nFrom: a@a.a\r\nTo: b@b.b\r\nDate: Thu, 12 Oct 2023 08:45:28 +0000\r\n\r\nhello world";
let content = FetchedMail::new_from_message(eml_codec::parse_message(rfc822)?.1);
let mut mv = MailView {
ids: &ids,
content,
meta: &meta,
flags: &flags,
add_seen: false,
};
let res_body = mv.filter(&ap)?;
let fattr = match res_body {
Body::Data(Data::Fetch {
seq_or_uid: _seq,
attributes: attr,
}) => Ok(attr),
_ => Err(anyhow!("Not a fetch body")),
}?;
assert_eq!(fattr.len(), 1);
let (sec, _orig, _data) = match &fattr[0] {
MessageAttribute::BodyExt {
section,
origin,
data,
} => Ok((section, origin, data)),
_ => Err(anyhow!("not a body ext message attribute")),
}?;
assert_eq!(sec.as_ref().unwrap(), &Section::Header(None));
Ok(())
}
/// Future automated test. We use lossy utf8 conversion + lowercase everything, /// Future automated test. We use lossy utf8 conversion + lowercase everything,
/// so this test might allow invalid results. But at least it allows us to quickly test a /// so this test might allow invalid results. But at least it allows us to quickly test a
/// large variety of emails. /// large variety of emails.
@ -1113,7 +1325,6 @@ mod tests {
//"tests/emails/dxflrs/0005_mail-parser-readme", //"tests/emails/dxflrs/0005_mail-parser-readme",
"tests/emails/dxflrs/0006_single-mime", "tests/emails/dxflrs/0006_single-mime",
"tests/emails/dxflrs/0007_raw_msg_in_rfc822", "tests/emails/dxflrs/0007_raw_msg_in_rfc822",
/* *** (STRANGE) RFC *** */ /* *** (STRANGE) RFC *** */
//"tests/emails/rfc/000", // must return text/enriched, we return text/plain //"tests/emails/rfc/000", // must return text/enriched, we return text/plain
//"tests/emails/rfc/001", // does not recognize the multipart/external-body, breaks the //"tests/emails/rfc/001", // does not recognize the multipart/external-body, breaks the
@ -1127,9 +1338,8 @@ mod tests {
//"tests/emails/thirdparty/001", // same //"tests/emails/thirdparty/001", // same
"tests/emails/thirdparty/002", // same "tests/emails/thirdparty/002", // same
/* *** LEGACY *** */
/* *** LEGACY *** */ //"tests/emails/legacy/000", // same issue with \r\r
//"tests/emails/legacy/000", // same issue with \r\r
]; ];
for pref in prefixes.iter() { for pref in prefixes.iter() {
@ -1139,7 +1349,9 @@ mod tests {
let message = eml_codec::parse_message(&txt).unwrap().1; let message = eml_codec::parse_message(&txt).unwrap().1;
let mut resp = Vec::new(); let mut resp = Vec::new();
MessageAttribute::Body(build_imap_email_struct(&message.child)?).encode(&mut resp).unwrap(); MessageAttribute::Body(build_imap_email_struct(&message.child)?)
.encode(&mut resp)
.unwrap();
let resp_str = String::from_utf8_lossy(&resp).to_lowercase(); let resp_str = String::from_utf8_lossy(&resp).to_lowercase();

16
src/login/mod.rs
View file

@ -7,7 +7,7 @@ use std::sync::Arc;
use anyhow::{anyhow, bail, Context, Result}; use anyhow::{anyhow, bail, Context, Result};
use async_trait::async_trait; use async_trait::async_trait;
use k2v_client::{ use k2v_client::{
BatchInsertOp, BatchReadOp, CausalValue, CausalityToken, Filter, K2vClient, K2vValue BatchInsertOp, BatchReadOp, CausalValue, CausalityToken, Filter, K2vClient, K2vValue,
}; };
use rand::prelude::*; use rand::prelude::*;
use rusoto_core::HttpClient; use rusoto_core::HttpClient;
@ -141,13 +141,13 @@ impl StorageCredentials {
self.aws_secret_access_key.clone(), self.aws_secret_access_key.clone(),
); );
let connector = hyper_rustls::HttpsConnectorBuilder::new() let connector = hyper_rustls::HttpsConnectorBuilder::new()
.with_native_roots() .with_native_roots()
.https_or_http() .https_or_http()
.enable_http1() .enable_http1()
.enable_http2() .enable_http2()
.build(); .build();
let client = HttpClient::from_connector(connector); let client = HttpClient::from_connector(connector);
Ok(S3Client::new_with( Ok(S3Client::new_with(
client, client,

8
src/mail/incoming.rs
View file

@ -450,10 +450,10 @@ impl EncryptedMessage {
let por = PutObjectRequest { let por = PutObjectRequest {
bucket: creds.storage.bucket.clone(), bucket: creds.storage.bucket.clone(),
key: format!("incoming/{}", gen_ident()), key: format!("incoming/{}", gen_ident()),
metadata: Some( metadata: Some(
[(MESSAGE_KEY.to_string(), key_header)] [(MESSAGE_KEY.to_string(), key_header)]
.into_iter() .into_iter()
.collect::<HashMap<_, _>>(), .collect::<HashMap<_, _>>(),
), ),
body: Some(self.encrypted_body.clone().into()), body: Some(self.encrypted_body.clone().into()),
..Default::default() ..Default::default()

6
src/mail/mailbox.rs
View file

@ -369,7 +369,7 @@ impl MailboxInternal {
// Save mail meta // Save mail meta
let meta = MailMeta { let meta = MailMeta {
internaldate: now_msec(), internaldate: now_msec(),
headers: mail.parsed.raw_headers.to_vec(), headers: mail.parsed.raw_headers.to_vec(),
message_key: message_key.clone(), message_key: message_key.clone(),
rfc822_size: mail.raw.len(), rfc822_size: mail.raw.len(),
}; };
@ -400,7 +400,7 @@ impl MailboxInternal {
futures::try_join!( futures::try_join!(
async { async {
// Delete mail body from S3 // Delete mail body from S3
let dor = DeleteObjectRequest{ let dor = DeleteObjectRequest {
bucket: self.bucket.clone(), bucket: self.bucket.clone(),
key: format!("{}/{}", self.mail_path, ident), key: format!("{}/{}", self.mail_path, ident),
..Default::default() ..Default::default()
@ -461,7 +461,7 @@ impl MailboxInternal {
futures::try_join!( futures::try_join!(
async { async {
// Copy mail body from S3 // Copy mail body from S3
let cor = CopyObjectRequest{ let cor = CopyObjectRequest {
bucket: self.bucket.clone(), bucket: self.bucket.clone(),
key: format!("{}/{}", self.mail_path, new_id), key: format!("{}/{}", self.mail_path, new_id),
copy_source: format!("{}/{}/{}", from.bucket, from.mail_path, source_id), copy_source: format!("{}/{}/{}", from.bucket, from.mail_path, source_id),

21
src/mail/user.rs
View file

@ -334,17 +334,22 @@ impl MailboxList {
} }
fn has_mailbox(&self, name: &str) -> bool { fn has_mailbox(&self, name: &str) -> bool {
matches!(self.0.get(name), Some(MailboxListEntry { matches!(
id_lww: (_, Some(_)), self.0.get(name),
.. Some(MailboxListEntry {
})) id_lww: (_, Some(_)),
..
})
)
} }
fn get_mailbox(&self, name: &str) -> Option<(ImapUidvalidity, Option<UniqueIdent>)> { fn get_mailbox(&self, name: &str) -> Option<(ImapUidvalidity, Option<UniqueIdent>)> {
self.0.get(name).map(|MailboxListEntry { self.0.get(name).map(
id_lww: (_, mailbox_id), |MailboxListEntry {
uidvalidity, id_lww: (_, mailbox_id),
}| (*uidvalidity, *mailbox_id)) uidvalidity,
}| (*uidvalidity, *mailbox_id),
)
} }
/// Ensures mailbox `name` maps to id `id`. /// Ensures mailbox `name` maps to id `id`.