Deuxfleurs/aerogramme
15
0
Fork 1
Code Issues 33 Pull requests 1 Projects 11 Releases 5 Wiki Activity

bump mail_parser

Browse source
This commit is contained in:
Quentin 2023-02-07 11:27:17 +01:00
parent ca385824b2
commit bb66f7cee3
Signed by: quentin
GPG key ID: E9602264D639FF68
4 changed files with 67 additions and 118 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
Cargo.lock generated
View file

@ -50,8 +50,8 @@ dependencies = [
"log", "log",
"mail-parser 0.4.8 (git+https://github.com/superboum/mail-parser?branch=feature/no_decode)", "mail-parser 0.4.8 (git+https://github.com/superboum/mail-parser?branch=feature/no_decode)",
"mail-parser 0.4.8 (git+https://github.com/superboum/mail-parser?rev=db61a03)", "mail-parser 0.4.8 (git+https://github.com/superboum/mail-parser?rev=db61a03)",
"mail-parser 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", "mail-parser 0.5.0",
"mail-parser 0.5.0 (git+https://github.com/stalwartlabs/mail-parser?branch=main)", "mail-parser 0.8.0",
"rand", "rand",
"rmp-serde", "rmp-serde",
"rpassword", "rpassword",
@ -1355,8 +1355,7 @@ dependencies = [
[[package]] [[package]]
name = "mail-parser" name = "mail-parser"
version = "0.5.0" version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "git+https://github.com/stalwartlabs/mail-parser?branch=main#bb0e3366daabec8d2f94afd22c661394f9ba7cbe"
checksum = "1e25e9e9cd1360538e0ca33499f12fd180f3611d7a8e1a50b7e4e43a4c5dd4b7"
dependencies = [ dependencies = [
"encoding_rs", "encoding_rs",
"serde", "serde",
@ -1364,11 +1363,11 @@ dependencies = [
[[package]] [[package]]
name = "mail-parser" name = "mail-parser"
version = "0.5.0" version = "0.8.0"
source = "git+https://github.com/stalwartlabs/mail-parser?branch=main#bb0e3366daabec8d2f94afd22c661394f9ba7cbe" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "80f330933d5287bcc21be7286850b353f22e7d66fa35d1c545ab030fd934b386"
dependencies = [ dependencies = [
"encoding_rs", "encoding_rs",
"serde",
] ]
[[package]] [[package]]

2
Cargo.toml
View file

@ -21,7 +21,7 @@ itertools = "0.10"
lazy_static = "1.4" lazy_static = "1.4"
ldap3 = { version = "0.10", default-features = false, features = ["tls"] } ldap3 = { version = "0.10", default-features = false, features = ["tls"] }
log = "0.4" log = "0.4"
mail-parser = "0.5" mail-parser = "0.8"
rusoto_core = "0.48.0" rusoto_core = "0.48.0"
rusoto_credential = "0.48.0" rusoto_credential = "0.48.0"
rusoto_s3 = "0.48" rusoto_s3 = "0.48"

166
src/imap/mailbox_view.rs
View file

@ -308,7 +308,7 @@ impl MailboxView {
))) )))
} }
FetchAttribute::Rfc822Text => { FetchAttribute::Rfc822Text => {
let rp = parsed.get_root_part(); let rp = parsed.root_part();
let r = parsed let r = parsed
.raw_message .raw_message
.get(rp.offset_body..rp.offset_end) .get(rp.offset_body..rp.offset_end)
@ -332,10 +332,10 @@ impl MailboxView {
attributes.push(MessageAttribute::Envelope(message_envelope(&parsed))) attributes.push(MessageAttribute::Envelope(message_envelope(&parsed)))
} }
FetchAttribute::Body => attributes.push(MessageAttribute::Body( FetchAttribute::Body => attributes.push(MessageAttribute::Body(
build_imap_email_struct(&parsed, parsed.get_root_part())?, build_imap_email_struct(&parsed, parsed.root_part())?,
)), )),
FetchAttribute::BodyStructure => attributes.push(MessageAttribute::Body( FetchAttribute::BodyStructure => attributes.push(MessageAttribute::Body(
build_imap_email_struct(&parsed, parsed.get_root_part())?, build_imap_email_struct(&parsed, parsed.root_part())?,
)), )),
FetchAttribute::BodyExt { FetchAttribute::BodyExt {
section, section,
@ -610,26 +610,26 @@ fn string_to_flag(f: &str) -> Option<Flag> {
//@FIXME return an error if the envelope is invalid instead of panicking //@FIXME return an error if the envelope is invalid instead of panicking
//@FIXME some fields must be defaulted if there are not set. //@FIXME some fields must be defaulted if there are not set.
fn message_envelope(msg: &mail_parser::Message<'_>) -> Envelope { fn message_envelope(msg: &mail_parser::Message<'_>) -> Envelope {
let from = convert_addresses(msg.get_from()).unwrap_or(vec![]); let from = convert_addresses(msg.from()).unwrap_or(vec![]);
Envelope { Envelope {
date: NString( date: NString(
msg.get_date() msg.date()
.map(|d| IString::try_from(d.to_iso8601()).unwrap()), .map(|d| IString::try_from(d.to_rfc3339()).unwrap()),
), ),
subject: NString( subject: NString(
msg.get_subject() msg.subject()
.map(|d| IString::try_from(d.to_string()).unwrap()), .map(|d| IString::try_from(d.to_string()).unwrap()),
), ),
from: from.clone(), from: from.clone(),
sender: convert_addresses(msg.get_sender()).unwrap_or(from.clone()), sender: convert_addresses(msg.sender()).unwrap_or(from.clone()),
reply_to: convert_addresses(msg.get_reply_to()).unwrap_or(from.clone()), reply_to: convert_addresses(msg.reply_to()).unwrap_or(from.clone()),
to: convert_addresses(msg.get_to()).unwrap_or(vec![]), to: convert_addresses(msg.to()).unwrap_or(vec![]),
cc: convert_addresses(msg.get_cc()).unwrap_or(vec![]), cc: convert_addresses(msg.cc()).unwrap_or(vec![]),
bcc: convert_addresses(msg.get_bcc()).unwrap_or(vec![]), bcc: convert_addresses(msg.bcc()).unwrap_or(vec![]),
in_reply_to: NString(None), // @TODO in_reply_to: NString(None), // @TODO
message_id: NString( message_id: NString(
msg.get_message_id() msg.message_id()
.map(|d| IString::try_from(d.to_string()).unwrap()), .map(|d| IString::try_from(d.to_string()).unwrap()),
), ),
} }
@ -642,12 +642,6 @@ fn convert_addresses(a: &mail_parser::HeaderValue<'_>) -> Option<Vec<Address>> {
Some(l.iter().map(|a| convert_address(a)).collect()) Some(l.iter().map(|a| convert_address(a)).collect())
} }
mail_parser::HeaderValue::Empty => None, mail_parser::HeaderValue::Empty => None,
mail_parser::HeaderValue::Collection(c) => Some(
c.iter()
.map(|l| convert_addresses(l).unwrap_or(vec![]))
.flatten()
.collect(),
),
_ => { _ => {
tracing::warn!("Invalid address header"); tracing::warn!("Invalid address header");
None None
@ -698,10 +692,10 @@ fn build_imap_email_struct<'a>(msg: &Message<'a>, part: &MessagePart<'a>) -> Res
match &part.body { match &part.body {
PartType::Multipart(parts) => { PartType::Multipart(parts) => {
let subtype = IString::try_from( let subtype = IString::try_from(
part.headers_rfc part.headers
.get(&RfcHeader::ContentType) .rfc(&RfcHeader::ContentType)
.ok_or(anyhow!("Content-Type is missing but required here."))? .ok_or(anyhow!("Content-Type is missing but required here."))?
.get_content_type() .content_type()
.c_subtype .c_subtype
.as_ref() .as_ref()
.ok_or(anyhow!("Content-Type invalid, missing subtype"))? .ok_or(anyhow!("Content-Type invalid, missing subtype"))?
@ -741,7 +735,7 @@ fn build_imap_email_struct<'a>(msg: &Message<'a>, part: &MessagePart<'a>) -> Res
// MUST be defined and hence has no default. But mail-parser does not make any // MUST be defined and hence has no default. But mail-parser does not make any
// difference between MIME and raw emails, hence raw emails have no subtypes. // difference between MIME and raw emails, hence raw emails have no subtypes.
let subtype = part let subtype = part
.get_content_type() .content_type()
.map(|h| h.c_subtype.as_ref()) .map(|h| h.c_subtype.as_ref())
.flatten() .flatten()
.map(|st| IString::try_from(st.to_string()).ok()) .map(|st| IString::try_from(st.to_string()).ok())
@ -770,7 +764,7 @@ fn build_imap_email_struct<'a>(msg: &Message<'a>, part: &MessagePart<'a>) -> Res
let (_, basic) = headers_to_basic_fields(&part, bp.len())?; let (_, basic) = headers_to_basic_fields(&part, bp.len())?;
let ct = part let ct = part
.get_content_type() .content_type()
.ok_or(anyhow!("Content-Type is missing but required here."))?; .ok_or(anyhow!("Content-Type is missing but required here."))?;
let type_ = IString::try_from(ct.c_type.as_ref().to_string()).map_err(|_| { let type_ = IString::try_from(ct.c_type.as_ref().to_string()).map_err(|_| {
@ -795,31 +789,26 @@ fn build_imap_email_struct<'a>(msg: &Message<'a>, part: &MessagePart<'a>) -> Res
extension: None, extension: None,
}) })
} }
PartType::Message(bp) => { PartType::Message(inner) => {
// @NOTE in some cases mail-parser does not parse the MessageAttachment but // @FIXME+BUG mail-parser does not handle ways when a MIME message contains
// provide it as raw body. By looking quickly at the code, it seems that the // a raw email and wrongly take its delimiter. The size and number of
// attachment is not parsed when mail-parser encounters some encoding problems. // lines returned in that case are wrong. A patch to mail-parser is
match &bp { // needed to fix this.
MessageAttachment::Parsed(inner) => { let (_, basic) = headers_to_basic_fields(&part, inner.raw_message.len())?;
// @FIXME+BUG mail-parser does not handle ways when a MIME message contains
// a raw email and wrongly take its delimiter. The size and number of
// lines returned in that case are wrong. A patch to mail-parser is
// needed to fix this.
let (_, basic) = headers_to_basic_fields(&part, inner.raw_message.len())?;
// We do not count the number of lines but the number of line // We do not count the number of lines but the number of line
// feeds to have the same behavior as Dovecot and Cyrus. // feeds to have the same behavior as Dovecot and Cyrus.
// 2 lines = 1 line feed. // 2 lines = 1 line feed.
let nol = inner.raw_message.iter().filter(|&c| c == &b'\n').count(); let nol = inner.raw_message.iter().filter(|&c| c == &b'\n').count();
Ok(BodyStructure::Single { Ok(BodyStructure::Single {
body: FetchBody { body: FetchBody {
basic, basic,
specific: SpecificFields::Message { specific: SpecificFields::Message {
envelope: message_envelope(inner), envelope: message_envelope(inner),
body_structure: Box::new(build_imap_email_struct( body_structure: Box::new(build_imap_email_struct(
&inner, &inner,
inner.get_root_part(), inner.root_part(),
)?), )?),
// @FIXME This solution is bad for 2 reasons: // @FIXME This solution is bad for 2 reasons:
@ -829,42 +818,10 @@ fn build_imap_email_struct<'a>(msg: &Message<'a>, part: &MessagePart<'a>) -> Res
// - It should be done during parsing, we are iterating twice on // - It should be done during parsing, we are iterating twice on
// the same data which results in some wastes. // the same data which results in some wastes.
number_of_lines: u32::try_from(nol)?, number_of_lines: u32::try_from(nol)?,
}, },
}, },
extension: None, extension: None,
}) })
}
MessageAttachment::Raw(raw_msg) => {
let (_, basic) = headers_to_basic_fields(&part, raw_msg.len())?;
let ct = part
.get_content_type()
.ok_or(anyhow!("Content-Type is missing but required here."))?;
let type_ =
IString::try_from(ct.c_type.as_ref().to_string()).map_err(|_| {
anyhow!("Unable to build IString from given Content-Type type given")
})?;
let subtype = IString::try_from(
ct.c_subtype
.as_ref()
.ok_or(anyhow!("Content-Type invalid, missing subtype"))?
.to_string(),
)
.map_err(|_| {
anyhow!("Unable to build IString from given Content-Type subtype given")
})?;
Ok(BodyStructure::Single {
body: FetchBody {
basic,
specific: SpecificFields::Basic { type_, subtype },
},
extension: None,
})
}
}
} }
} }
} }
@ -896,7 +853,7 @@ struct SpecialAttrs<'a> {
fn attrs_to_params<'a>(bp: &impl MimeHeaders<'a>) -> (SpecialAttrs, Vec<(IString, IString)>) { fn attrs_to_params<'a>(bp: &impl MimeHeaders<'a>) -> (SpecialAttrs, Vec<(IString, IString)>) {
// Try to extract Content-Type attributes from headers // Try to extract Content-Type attributes from headers
let attrs = match bp let attrs = match bp
.get_content_type() .content_type()
.map(|c| c.attributes.as_ref()) .map(|c| c.attributes.as_ref())
.flatten() .flatten()
{ {
@ -942,13 +899,13 @@ fn headers_to_basic_fields<'a>(
parameter_list, parameter_list,
id: NString( id: NString(
bp.get_content_id() bp.content_id()
.map(|ci| IString::try_from(ci.to_string()).ok()) .map(|ci| IString::try_from(ci.to_string()).ok())
.flatten(), .flatten(),
), ),
description: NString( description: NString(
bp.get_content_description() bp.content_description()
.map(|cd| IString::try_from(cd.to_string()).ok()) .map(|cd| IString::try_from(cd.to_string()).ok())
.flatten(), .flatten(),
), ),
@ -959,7 +916,7 @@ fn headers_to_basic_fields<'a>(
* Content-Transfer-Encoding header field is not present. * Content-Transfer-Encoding header field is not present.
*/ */
content_transfer_encoding: bp content_transfer_encoding: bp
.get_content_transfer_encoding() .content_transfer_encoding()
.map(|h| IString::try_from(h.to_string()).ok()) .map(|h| IString::try_from(h.to_string()).ok())
.flatten() .flatten()
.unwrap_or(unchecked_istring("7bit")), .unwrap_or(unchecked_istring("7bit")),
@ -976,7 +933,7 @@ fn get_message_section<'a>(
) -> Result<Cow<'a, [u8]>> { ) -> Result<Cow<'a, [u8]>> {
match section { match section {
Some(FetchSection::Text(None)) => { Some(FetchSection::Text(None)) => {
let rp = parsed.get_root_part(); let rp = parsed.root_part();
Ok(parsed Ok(parsed
.raw_message .raw_message
.get(rp.offset_body..rp.offset_end) .get(rp.offset_body..rp.offset_end)
@ -987,7 +944,7 @@ fn get_message_section<'a>(
} }
Some(FetchSection::Text(Some(part))) => { Some(FetchSection::Text(Some(part))) => {
map_subpart_msg(parsed, part.0.as_slice(), |part_msg| { map_subpart_msg(parsed, part.0.as_slice(), |part_msg| {
let rp = part_msg.get_root_part(); let rp = part_msg.root_part();
Ok(part_msg Ok(part_msg
.raw_message .raw_message
.get(rp.offset_body..rp.offset_end) .get(rp.offset_body..rp.offset_end)
@ -1002,7 +959,7 @@ fn get_message_section<'a>(
parsed, parsed,
part.as_ref().map(|p| p.0.as_slice()).unwrap_or(&[]), part.as_ref().map(|p| p.0.as_slice()).unwrap_or(&[]),
|part_msg| { |part_msg| {
let rp = part_msg.get_root_part(); let rp = part_msg.root_part();
Ok(part_msg Ok(part_msg
.raw_message .raw_message
.get(..rp.offset_body) .get(..rp.offset_body)
@ -1031,13 +988,13 @@ fn get_message_section<'a>(
part.as_ref().map(|p| p.0.as_slice()).unwrap_or(&[]), part.as_ref().map(|p| p.0.as_slice()).unwrap_or(&[]),
|part_msg| { |part_msg| {
let mut ret = vec![]; let mut ret = vec![];
for (hn, hv) in part_msg.get_raw_headers() { for (hn, hv) in part_msg.headers_raw() {
if fields if fields
.as_slice() .as_slice()
.iter() .iter()
.any(|x| (*x == hn.as_str().as_bytes()) ^ invert) .any(|x| (*x == hn.as_bytes()) ^ invert)
{ {
ret.extend(hn.as_str().as_bytes()); ret.extend(hn.as_bytes());
ret.extend(b": "); ret.extend(b": ");
ret.extend(hv.as_bytes()); ret.extend(hv.as_bytes());
} }
@ -1051,18 +1008,17 @@ fn get_message_section<'a>(
let bytes = match &part.body { let bytes = match &part.body {
PartType::Text(p) | PartType::Html(p) => p.as_bytes().to_vec(), PartType::Text(p) | PartType::Html(p) => p.as_bytes().to_vec(),
PartType::Binary(p) | PartType::InlineBinary(p) => p.to_vec(), PartType::Binary(p) | PartType::InlineBinary(p) => p.to_vec(),
PartType::Message(MessageAttachment::Raw(r)) => r.to_vec(), PartType::Message(p) => p.raw_message.to_vec(),
PartType::Message(MessageAttachment::Parsed(p)) => p.raw_message.to_vec(),
PartType::Multipart(_) => bail!("Multipart part has no body"), PartType::Multipart(_) => bail!("Multipart part has no body"),
}; };
Ok(bytes.into()) Ok(bytes.into())
}), }),
Some(FetchSection::Mime(part)) => map_subpart(parsed, part.0.as_slice(), |msg, part| { Some(FetchSection::Mime(part)) => map_subpart(parsed, part.0.as_slice(), |msg, part| {
let mut ret = vec![]; let mut ret = vec![];
for (name, body) in part.headers_raw.iter() { for head in part.headers.iter() {
ret.extend(name.as_str().as_bytes()); ret.extend(head.name.as_str().as_bytes());
ret.extend(b": "); ret.extend(b": ");
ret.extend(&msg.raw_message[body.start..body.end]); ret.extend(&msg.raw_message[head.offset_start..head.offset_end]);
} }
ret.extend(b"\r\n"); ret.extend(b"\r\n");
Ok(ret.into()) Ok(ret.into())
@ -1082,11 +1038,8 @@ where
.parts .parts
.get(path[0].get() as usize - 1) .get(path[0].get() as usize - 1)
.ok_or(anyhow!("No such subpart: {}", path[0]))?; .ok_or(anyhow!("No such subpart: {}", path[0]))?;
if let PartType::Message(msg_attch) = &part.body { if let PartType::Message(msg_attach) = &part.body {
let part_msg = msg_attch map_subpart_msg(&msg_attach, &path[1..], f)
.get_message()
.ok_or(anyhow!("Cannot parse subpart: {}", path[0]))?;
map_subpart_msg(&part_msg, &path[1..], f)
} else { } else {
bail!("Subpart is not a message: {}", path[0]); bail!("Subpart is not a message: {}", path[0]);
} }
@ -1107,11 +1060,8 @@ where
if path.len() == 1 { if path.len() == 1 {
f(msg, part) f(msg, part)
} else { } else {
if let PartType::Message(msg_attch) = &part.body { if let PartType::Message(msg_attach) = &part.body {
let part_msg = msg_attch map_subpart(&msg_attach, &path[1..], f)
.get_message()
.ok_or(anyhow!("Cannot parse subpart: {}", path[0]))?;
map_subpart(&part_msg, &path[1..], f)
} else { } else {
bail!("Subpart is not a message: {}", path[0]); bail!("Subpart is not a message: {}", path[0]);
} }

4
src/mail/mailbox.rs
View file

@ -308,7 +308,7 @@ impl MailboxInternal {
}, },
async { async {
// Save mail meta // Save mail meta
let mail_root = mail.parsed.get_root_part(); let mail_root = mail.parsed.root_part();
let meta = MailMeta { let meta = MailMeta {
internaldate: now_msec(), internaldate: now_msec(),
headers: mail.raw[..mail_root.offset_body].to_vec(), headers: mail.raw[..mail_root.offset_body].to_vec(),
@ -359,7 +359,7 @@ impl MailboxInternal {
}, },
async { async {
// Save mail meta // Save mail meta
let mail_root = mail.parsed.get_root_part(); let mail_root = mail.parsed.root_part();
let meta = MailMeta { let meta = MailMeta {
internaldate: now_msec(), internaldate: now_msec(),
headers: mail.raw[..mail_root.offset_body].to_vec(), headers: mail.raw[..mail_root.offset_body].to_vec(),