Merge pull request 'bug/thunderbird' (#68) from bug/thunderbird into main

Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/aerogramme/pulls/68

9 files changed, 147 insertions, 111 deletions

diff --git a/src/bayou.rs b/src/bayou.rs
index c6a7ac0..d77e9dc 100644
--- a/src/bayou.rs
+++ b/src/bayou.rs
@@ -152,7 +152,7 @@ impl<S: BayouState> Bayou<S> {
             match &val[0] {
                 storage::Alternative::Value(v) => {
                     let op = open_deserialize::<S::Op>(v, &self.key)?;
-                    debug!("(sync) operation {}: {:?}", sort_key, op);
+                    tracing::trace!("(sync) operation {}: {:?}", sort_key, op);
                     ops.push((ts, op));
                 }
                 storage::Alternative::Tombstone => {
diff --git a/src/imap/attributes.rs b/src/imap/attributes.rs
index 7a55632..cf7cb52 100644
--- a/src/imap/attributes.rs
+++ b/src/imap/attributes.rs
@@ -1,4 +1,4 @@
-use imap_codec::imap_types::fetch::{MacroOrMessageDataItemNames, MessageDataItemName};
+use imap_codec::imap_types::fetch::{MacroOrMessageDataItemNames, MessageDataItemName, Section};
 
 /// Internal decisions based on fetched attributes
 /// passed by the client
@@ -36,14 +36,25 @@ impl AttributesProxy {
 
     pub fn need_body(&self) -> bool {
         self.attrs.iter().any(|x| {
-            matches!(
-                x,
+            match x {
                 MessageDataItemName::Body
-                    | MessageDataItemName::BodyExt { .. }
-                    | MessageDataItemName::Rfc822
-                    | MessageDataItemName::Rfc822Text
-                    | MessageDataItemName::BodyStructure
-            )
+                | MessageDataItemName::Rfc822
+                | MessageDataItemName::Rfc822Text
+                | MessageDataItemName::BodyStructure => true,
+
+                MessageDataItemName::BodyExt {
+                    section: Some(section),
+                    partial: _,
+                    peek: _,
+                } => match section {
+                    Section::Header(None)
+                    | Section::HeaderFields(None, _)
+                    | Section::HeaderFieldsNot(None, _) => false,
+                    _ => true,
+                },
+                MessageDataItemName::BodyExt { .. } => true,
+                _ => false,
+            }
         })
     }
 }
diff --git a/src/imap/index.rs b/src/imap/index.rs
index 3ca5562..4853374 100644
--- a/src/imap/index.rs
+++ b/src/imap/index.rs
@@ -1,6 +1,6 @@
 use std::num::NonZeroU32;
 
-use anyhow::{anyhow, Context, Result};
+use anyhow::{anyhow, Result};
 use imap_codec::imap_types::sequence::{self, SeqOrUid, Sequence, SequenceSet};
 
 use crate::mail::uidindex::{ImapUid, UidIndex};
@@ -62,7 +62,7 @@ impl<'a> Index<'a> {
             return vec![];
         }
         let iter_strat = sequence::Strategy::Naive {
-            largest: self.last().expect("imap index is not empty").uid,
+            largest: self.last().expect("The mailbox is not empty").uid,
         };
         let mut unroll_seq = sequence_set.iter(iter_strat).collect::<Vec<_>>();
         unroll_seq.sort();
@@ -80,10 +80,6 @@ impl<'a> Index<'a> {
                 .partition_point(|mail_idx| &mail_idx.uid < start_seq);
             &self.imap_index[start_idx..]
         };
-        println!(
-            "win: {:?}",
-            imap_idx.iter().map(|midx| midx.uid).collect::<Vec<_>>()
-        );
 
         let mut acc = vec![];
         for wanted_uid in unroll_seq.iter() {
@@ -104,17 +100,25 @@ impl<'a> Index<'a> {
     }
 
     pub fn fetch_on_id(&'a self, sequence_set: &SequenceSet) -> Result<Vec<&'a MailIndex<'a>>> {
+        if self.imap_index.is_empty() {
+            return Ok(vec![]);
+        }
         let iter_strat = sequence::Strategy::Naive {
-            largest: self.last().context("The mailbox is empty")?.uid,
+            largest: NonZeroU32::try_from(self.imap_index.len() as u32)?,
         };
-        sequence_set
+        let mut acc = sequence_set
             .iter(iter_strat)
             .map(|wanted_id| {
                 self.imap_index
                     .get((wanted_id.get() as usize) - 1)
                     .ok_or(anyhow!("Mail not found"))
             })
-            .collect::<Result<Vec<_>>>()
+            .collect::<Result<Vec<_>>>()?;
+
+        // Sort the result to be consistent with UID
+        acc.sort_by(|a, b| a.i.cmp(&b.i));
+
+        Ok(acc)
     }
 
     pub fn fetch(
diff --git a/src/imap/mail_view.rs b/src/imap/mail_view.rs
index 7da21c4..eeb6b4b 100644
--- a/src/imap/mail_view.rs
+++ b/src/imap/mail_view.rs
@@ -27,15 +27,12 @@ use crate::imap::response::Body;
 
 pub struct MailView<'a> {
     pub in_idx: &'a MailIndex<'a>,
-    pub query_result: &'a QueryResult<'a>,
+    pub query_result: &'a QueryResult,
     pub content: FetchedMail<'a>,
 }
 
 impl<'a> MailView<'a> {
-    pub fn new(
-        query_result: &'a QueryResult<'a>,
-        in_idx: &'a MailIndex<'a>,
-    ) -> Result<MailView<'a>> {
+    pub fn new(query_result: &'a QueryResult, in_idx: &'a MailIndex<'a>) -> Result<MailView<'a>> {
         Ok(Self {
             in_idx,
             query_result,
@@ -74,7 +71,12 @@ impl<'a> MailView<'a> {
                 MessageDataItemName::Rfc822Size => self.rfc_822_size(),
                 MessageDataItemName::Rfc822Header => self.rfc_822_header(),
                 MessageDataItemName::Rfc822Text => self.rfc_822_text(),
-                MessageDataItemName::Rfc822 => self.rfc822(),
+                MessageDataItemName::Rfc822 => {
+                    if self.is_not_yet_seen() {
+                        seen = SeenFlag::MustAdd;
+                    }
+                    self.rfc822()
+                }
                 MessageDataItemName::Envelope => Ok(self.envelope()),
                 MessageDataItemName::Body => self.body(),
                 MessageDataItemName::BodyStructure => self.body_structure(),
@@ -180,15 +182,22 @@ impl<'a> MailView<'a> {
     fn body(&self) -> Result<MessageDataItem<'static>> {
         Ok(MessageDataItem::Body(mime_view::bodystructure(
             self.content.as_msg()?.child.as_ref(),
+            false,
         )?))
     }
 
     fn body_structure(&self) -> Result<MessageDataItem<'static>> {
-        Ok(MessageDataItem::Body(mime_view::bodystructure(
+        Ok(MessageDataItem::BodyStructure(mime_view::bodystructure(
             self.content.as_msg()?.child.as_ref(),
+            true,
         )?))
     }
 
+    fn is_not_yet_seen(&self) -> bool {
+        let seen_flag = Flag::Seen.to_string();
+        !self.in_idx.flags.iter().any(|x| *x == seen_flag)
+    }
+
     /// maps to BODY[<section>]<<partial>> and BODY.PEEK[<section>]<<partial>>
     /// peek does not implicitly set the \Seen flag
     /// eg. BODY[HEADER.FIELDS (DATE FROM)]
@@ -201,8 +210,7 @@ impl<'a> MailView<'a> {
     ) -> Result<(MessageDataItem<'static>, SeenFlag)> {
         // Manage Seen flag
         let mut seen = SeenFlag::DoNothing;
-        let seen_flag = Flag::Seen.to_string();
-        if !peek && !self.in_idx.flags.iter().any(|x| *x == seen_flag) {
+        if !peek && self.is_not_yet_seen() {
             // Add \Seen flag
             //self.mailbox.add_flags(uuid, &[seen_flag]).await?;
             seen = SeenFlag::MustAdd;
diff --git a/src/imap/mailbox_view.rs b/src/imap/mailbox_view.rs
index 77fe7f7..513567f 100644
--- a/src/imap/mailbox_view.rs
+++ b/src/imap/mailbox_view.rs
@@ -130,6 +130,8 @@ impl MailboxView {
         data.extend(self.flags_status()?.into_iter());
         data.push(self.uidvalidity_status()?);
         data.push(self.uidnext_status()?);
+        self.unseen_first_status()?
+            .map(|unseen_status| data.push(unseen_status));
 
         Ok(data)
     }
@@ -257,6 +259,7 @@ impl MailboxView {
             true => QueryScope::Full,
             _ => QueryScope::Partial,
         };
+        tracing::debug!("Query scope {:?}", query_scope);
         let idx = self.index()?;
         let mail_idx_list = idx.fetch(sequence_set, *is_uid_fetch)?;
 
@@ -400,6 +403,27 @@ impl MailboxView {
         Ok(Body::Data(Data::Recent(self.recent()?)))
     }
 
+    fn unseen_first_status(&self) -> Result<Option<Body<'static>>> {
+        Ok(self
+            .unseen_first()?
+            .map(|unseen_id| {
+                Status::ok(None, Some(Code::Unseen(unseen_id)), "First unseen.").map(Body::Status)
+            })
+            .transpose()?)
+    }
+
+    fn unseen_first(&self) -> Result<Option<NonZeroU32>> {
+        Ok(self
+            .0
+            .snapshot
+            .table
+            .values()
+            .enumerate()
+            .find(|(_i, (_imap_uid, flags))| !flags.contains(&"\\Seen".to_string()))
+            .map(|(i, _)| NonZeroU32::try_from(i as u32 + 1))
+            .transpose()?)
+    }
+
     pub(crate) fn recent(&self) -> Result<u32> {
         let recent = self
             .0
@@ -521,7 +545,6 @@ mod tests {
         let rfc822 = b"Subject: hello\r\nFrom: a@a.a\r\nTo: b@b.b\r\nDate: Thu, 12 Oct 2023 08:45:28 +0000\r\n\r\nhello world";
         let qr = QueryResult::FullResult {
             uuid: mail_in_idx.uuid.clone(),
-            index: &index_entry,
             metadata: meta,
             content: rfc822.to_vec(),
         };
@@ -596,6 +619,7 @@ mod tests {
                 seq: NonZeroU32::new(1).unwrap(),
                 items: NonEmptyVec::from(MessageDataItem::Body(mime_view::bodystructure(
                     &message.child,
+                    false,
                 )?)),
             });
             let test_bytes = ResponseCodec::new().encode(&test_repr).dump();
diff --git a/src/imap/mime_view.rs b/src/imap/mime_view.rs
index cf6c751..8fc043b 100644
--- a/src/imap/mime_view.rs
+++ b/src/imap/mime_view.rs
@@ -4,7 +4,10 @@ use std::num::NonZeroU32;
 
 use anyhow::{anyhow, bail, Result};
 
-use imap_codec::imap_types::body::{BasicFields, Body as FetchBody, BodyStructure, SpecificFields};
+use imap_codec::imap_types::body::{
+    BasicFields, Body as FetchBody, BodyStructure, MultiPartExtensionData, SinglePartExtensionData,
+    SpecificFields,
+};
 use imap_codec::imap_types::core::{AString, IString, NString, NonEmptyVec};
 use imap_codec::imap_types::fetch::{Part as FetchPart, Section as FetchSection};
 
@@ -78,8 +81,8 @@ pub fn body_ext<'a>(
 ///                                   | parameter list
 /// b OK Fetch completed (0.001 + 0.000 secs).
 /// ```
-pub fn bodystructure(part: &AnyPart) -> Result<BodyStructure<'static>> {
-    NodeMime(part).structure()
+pub fn bodystructure(part: &AnyPart, is_ext: bool) -> Result<BodyStructure<'static>> {
+    NodeMime(part).structure(is_ext)
 }
 
 /// NodeMime
@@ -118,12 +121,12 @@ impl<'a> NodeMime<'a> {
         }
     }
 
-    fn structure(&self) -> Result<BodyStructure<'static>> {
+    fn structure(&self, is_ext: bool) -> Result<BodyStructure<'static>> {
         match self.0 {
-            AnyPart::Txt(x) => NodeTxt(self, x).structure(),
-            AnyPart::Bin(x) => NodeBin(self, x).structure(),
-            AnyPart::Mult(x) => NodeMult(self, x).structure(),
-            AnyPart::Msg(x) => NodeMsg(self, x).structure(),
+            AnyPart::Txt(x) => NodeTxt(self, x).structure(is_ext),
+            AnyPart::Bin(x) => NodeBin(self, x).structure(is_ext),
+            AnyPart::Mult(x) => NodeMult(self, x).structure(is_ext),
+            AnyPart::Msg(x) => NodeMsg(self, x).structure(is_ext),
         }
     }
 }
@@ -359,7 +362,7 @@ impl<'a> SelectedMime<'a> {
 // ---------------------------
 struct NodeMsg<'a>(&'a NodeMime<'a>, &'a composite::Message<'a>);
 impl<'a> NodeMsg<'a> {
-    fn structure(&self) -> Result<BodyStructure<'static>> {
+    fn structure(&self, is_ext: bool) -> Result<BodyStructure<'static>> {
         let basic = SelectedMime(self.0 .0).basic_fields()?;
 
         Ok(BodyStructure::Single {
@@ -367,17 +370,23 @@ impl<'a> NodeMsg<'a> {
                 basic,
                 specific: SpecificFields::Message {
                     envelope: Box::new(ImfView(&self.1.imf).message_envelope()),
-                    body_structure: Box::new(NodeMime(&self.1.child).structure()?),
+                    body_structure: Box::new(NodeMime(&self.1.child).structure(is_ext)?),
                     number_of_lines: nol(self.1.raw_part),
                 },
             },
-            extension_data: None,
+            extension_data: match is_ext {
+                true => Some(SinglePartExtensionData {
+                    md5: NString(None),
+                    tail: None,
+                }),
+                _ => None,
+            },
         })
     }
 }
 struct NodeMult<'a>(&'a NodeMime<'a>, &'a composite::Multipart<'a>);
 impl<'a> NodeMult<'a> {
-    fn structure(&self) -> Result<BodyStructure<'static>> {
+    fn structure(&self, is_ext: bool) -> Result<BodyStructure<'static>> {
         let itype = &self.1.mime.interpreted_type;
         let subtype = IString::try_from(itype.subtype.to_string())
             .unwrap_or(unchecked_istring("alternative"));
@@ -386,7 +395,7 @@ impl<'a> NodeMult<'a> {
             .1
             .children
             .iter()
-            .filter_map(|inner| NodeMime(&inner).structure().ok())
+            .filter_map(|inner| NodeMime(&inner).structure(is_ext).ok())
             .collect::<Vec<_>>();
 
         NonEmptyVec::validate(&inner_bodies)?;
@@ -395,20 +404,22 @@ impl<'a> NodeMult<'a> {
         Ok(BodyStructure::Multi {
             bodies,
             subtype,
-            extension_data: None,
-            /*Some(MultipartExtensionData {
-            parameter_list: vec![],
-            disposition: None,
-            language: None,
-            location: None,
-            extension: vec![],
-            })*/
+            extension_data: match is_ext {
+                true => Some(MultiPartExtensionData {
+                    parameter_list: vec![(
+                        IString::try_from("boundary").unwrap(),
+                        IString::try_from(self.1.mime.interpreted_type.boundary.to_string())?,
+                    )],
+                    tail: None,
+                }),
+                _ => None,
+            },
         })
     }
 }
 struct NodeTxt<'a>(&'a NodeMime<'a>, &'a discrete::Text<'a>);
 impl<'a> NodeTxt<'a> {
-    fn structure(&self) -> Result<BodyStructure<'static>> {
+    fn structure(&self, is_ext: bool) -> Result<BodyStructure<'static>> {
         let mut basic = SelectedMime(self.0 .0).basic_fields()?;
 
         // Get the interpreted content type, set it
@@ -435,14 +446,20 @@ impl<'a> NodeTxt<'a> {
                     number_of_lines: nol(self.1.body),
                 },
             },
-            extension_data: None,
+            extension_data: match is_ext {
+                true => Some(SinglePartExtensionData {
+                    md5: NString(None),
+                    tail: None,
+                }),
+                _ => None,
+            },
         })
     }
 }
 
 struct NodeBin<'a>(&'a NodeMime<'a>, &'a discrete::Binary<'a>);
 impl<'a> NodeBin<'a> {
-    fn structure(&self) -> Result<BodyStructure<'static>> {
+    fn structure(&self, is_ext: bool) -> Result<BodyStructure<'static>> {
         let basic = SelectedMime(self.0 .0).basic_fields()?;
 
         let default = mime::r#type::NaiveType {
@@ -465,7 +482,13 @@ impl<'a> NodeBin<'a> {
                 basic,
                 specific: SpecificFields::Basic { r#type, subtype },
             },
-            extension_data: None,
+            extension_data: match is_ext {
+                true => Some(SinglePartExtensionData {
+                    md5: NString(None),
+                    tail: None,
+                }),
+                _ => None,
+            },
         })
     }
 }
diff --git a/src/imap/search.rs b/src/imap/search.rs
index 22afd0c..c4888d0 100644
--- a/src/imap/search.rs
+++ b/src/imap/search.rs
@@ -134,7 +134,7 @@ impl<'a> Criteria<'a> {
     pub fn filter_on_query<'b>(
         &self,
         midx_list: &[&'b MailIndex<'b>],
-        query_result: &'b Vec<QueryResult<'b>>,
+        query_result: &'b Vec<QueryResult>,
     ) -> Result<Vec<&'b MailIndex<'b>>> {
         Ok(midx_list
             .iter()
diff --git a/src/mail/mailbox.rs b/src/mail/mailbox.rs
index 7eed34f..aab200b 100644
--- a/src/mail/mailbox.rs
+++ b/src/mail/mailbox.rs
@@ -486,7 +486,7 @@ fn dump(uid_index: &Bayou<UidIndex>) {
 
 /// The metadata of a message that is stored in K2V
 /// at pk = mail/<mailbox uuid>, sk = <message uuid>
-#[derive(Serialize, Deserialize)]
+#[derive(Debug, Serialize, Deserialize)]
 pub struct MailMeta {
     /// INTERNALDATE field (milliseconds since epoch)
     pub internaldate: u64,
diff --git a/src/mail/query.rs b/src/mail/query.rs
index 91bd6c1..0838800 100644
--- a/src/mail/query.rs
+++ b/src/mail/query.rs
@@ -1,9 +1,8 @@
 use super::mailbox::MailMeta;
 use super::snapshot::FrozenMailbox;
-use super::uidindex::IndexEntry;
 use super::unique_ident::UniqueIdent;
-use anyhow::{anyhow, Result};
-use futures::stream::{FuturesUnordered, StreamExt};
+use anyhow::Result;
+use futures::stream::{FuturesOrdered, StreamExt};
 
 /// Query is in charge of fetching efficiently
 /// requested data for a list of emails
@@ -13,7 +12,7 @@ pub struct Query<'a, 'b> {
     pub scope: QueryScope,
 }
 
-#[allow(dead_code)]
+#[derive(Debug)]
 pub enum QueryScope {
     Index,
     Partial,
@@ -30,9 +29,13 @@ impl QueryScope {
 }
 
 impl<'a, 'b> Query<'a, 'b> {
-    pub async fn fetch(&self) -> Result<Vec<QueryResult<'a>>> {
+    pub async fn fetch(&self) -> Result<Vec<QueryResult>> {
         match self.scope {
-            QueryScope::Index => self.index(),
+            QueryScope::Index => Ok(self
+                .emails
+                .iter()
+                .map(|&uuid| QueryResult::IndexResult { uuid })
+                .collect()),
             QueryScope::Partial => self.partial().await,
             QueryScope::Full => self.full().await,
         }
@@ -40,31 +43,14 @@ impl<'a, 'b> Query<'a, 'b> {
 
     // --- functions below are private *for reasons*
 
-    fn index(&self) -> Result<Vec<QueryResult<'a>>> {
-        self.emails
-            .iter()
-            .map(|uuid| {
-                self.frozen
-                    .snapshot
-                    .table
-                    .get(uuid)
-                    .map(|index| QueryResult::IndexResult { uuid: *uuid, index })
-                    .ok_or(anyhow!("missing email in index"))
-            })
-            .collect::<Result<Vec<_>, _>>()
-    }
-
-    async fn partial(&self) -> Result<Vec<QueryResult<'a>>> {
+    async fn partial(&self) -> Result<Vec<QueryResult>> {
         let meta = self.frozen.mailbox.fetch_meta(self.emails).await?;
         let result = meta
             .into_iter()
-            .zip(self.index()?)
-            .map(|(metadata, index)| {
-                index
-                    .into_partial(metadata)
-                    .expect("index to be IndexResult")
-            })
+            .zip(self.emails.iter())
+            .map(|(metadata, &uuid)| QueryResult::PartialResult { uuid, metadata })
             .collect::<Vec<_>>();
+
         Ok(result)
     }
 
@@ -72,7 +58,7 @@ impl<'a, 'b> Query<'a, 'b> {
     /// AND GENERATE SO MUCH NETWORK TRAFFIC.
     /// THIS FUNCTION SHOULD BE REWRITTEN, FOR EXAMPLE WITH
     /// SOMETHING LIKE AN ITERATOR
-    async fn full(&self) -> Result<Vec<QueryResult<'a>>> {
+    async fn full(&self) -> Result<Vec<QueryResult>> {
         let meta_list = self.partial().await?;
         meta_list
             .into_iter()
@@ -91,7 +77,7 @@ impl<'a, 'b> Query<'a, 'b> {
 
                 Ok(meta.into_full(content).expect("meta to be PartialResult"))
             })
-            .collect::<FuturesUnordered<_>>()
+            .collect::<FuturesOrdered<_>>()
             .collect::<Vec<_>>()
             .await
             .into_iter()
@@ -99,24 +85,22 @@ impl<'a, 'b> Query<'a, 'b> {
     }
 }
 
-pub enum QueryResult<'a> {
+#[derive(Debug)]
+pub enum QueryResult {
     IndexResult {
         uuid: UniqueIdent,
-        index: &'a IndexEntry,
     },
     PartialResult {
         uuid: UniqueIdent,
-        index: &'a IndexEntry,
         metadata: MailMeta,
     },
     FullResult {
         uuid: UniqueIdent,
-        index: &'a IndexEntry,
         metadata: MailMeta,
         content: Vec<u8>,
     },
 }
-impl<'a> QueryResult<'a> {
+impl QueryResult {
     pub fn uuid(&self) -> &UniqueIdent {
         match self {
             Self::IndexResult { uuid, .. } => uuid,
@@ -125,16 +109,7 @@ impl<'a> QueryResult<'a> {
         }
     }
 
-    #[allow(dead_code)]
-    pub fn index(&self) -> &IndexEntry {
-        match self {
-            Self::IndexResult { index, .. } => index,
-            Self::PartialResult { index, .. } => index,
-            Self::FullResult { index, .. } => index,
-        }
-    }
-
-    pub fn metadata(&'a self) -> Option<&'a MailMeta> {
+    pub fn metadata(&self) -> Option<&MailMeta> {
         match self {
             Self::IndexResult { .. } => None,
             Self::PartialResult { metadata, .. } => Some(metadata),
@@ -143,7 +118,7 @@ impl<'a> QueryResult<'a> {
     }
 
     #[allow(dead_code)]
-    pub fn content(&'a self) -> Option<&'a [u8]> {
+    pub fn content(&self) -> Option<&[u8]> {
         match self {
             Self::FullResult { content, .. } => Some(content),
             _ => None,
@@ -152,24 +127,15 @@ impl<'a> QueryResult<'a> {
 
     fn into_partial(self, metadata: MailMeta) -> Option<Self> {
         match self {
-            Self::IndexResult { uuid, index } => Some(Self::PartialResult {
-                uuid,
-                index,
-                metadata,
-            }),
+            Self::IndexResult { uuid } => Some(Self::PartialResult { uuid, metadata }),
             _ => None,
         }
     }
 
     fn into_full(self, content: Vec<u8>) -> Option<Self> {
         match self {
-            Self::PartialResult {
-                uuid,
-                index,
-                metadata,
-            } => Some(Self::FullResult {
+            Self::PartialResult { uuid, metadata } => Some(Self::FullResult {
                 uuid,
-                index,
                 metadata,
                 content,
             }),