fix readme

README.md

@@ -18,19 +18,12 @@ A resilient & standards-compliant open-source IMAP server with built-in encrypti
 
 ## Roadmap
 
-  - ✅ 0.1 Better emails parsing (july '23, see [eml-codec](https://git.deuxfleurs.fr/Deuxfleurs/eml-codec)).
-  - ✅ 0.2 Support of IMAP4. (~january '24).
-  - ⌛0.3 CalDAV support. (~february '24).
+  - ✅ 0.1 Better emails parsing.
+  - ✅ 0.2 IMAP4 support.
+  - ✅ 0.3 CalDAV support.
   - ⌛0.4 CardDAV support.
-  - ⌛0.5 Public beta.
-
-## A note about cargo2nix
-
-Currently, you must edit Cargo.nix by hand after running `cargo2nix`.
-Find the `tokio` dependency declaration. 
-Look at tokio's dependencies, the `tracing` is disable through a `if false` logic.
-Activate it by replacing the condition with `if true`.
-
+  - ⌛0.5 Internals rework.
+  - ⌛0.6 Public beta.
 
 ## Sponsors and funding
 

aero-collections/src/calendar/mod.rs

@@ -177,6 +177,10 @@ impl CalendarInternal {
             .iter()
             .filter_map(|t: &Token| davstate.change.get(t))
             .map(|s| s.clone())
+            .filter(|s| match s {
+                SyncChange::Ok((filename, _)) => davstate.idx_by_filename.get(filename).is_some(),
+                SyncChange::NotFound(filename) => davstate.idx_by_filename.get(filename).is_none(),
+            })
             .collect();
 
         let token = self.current_token().await?;

aero-dav/src/syncencoder.rs

@@ -128,7 +128,6 @@ mod tests {
         src.qwrite(&mut writer).await.expect("xml serialization");
         tokio_buffer.flush().await.expect("tokio buffer flush");
         let got = std::str::from_utf8(buffer.as_slice()).unwrap();
-        println!("{:?}", got);
 
         // deserialize
         let mut rdr = Reader::new(quick_xml::NsReader::from_reader(got.as_bytes()))

aero-proto/src/dav/controller.rs

@@ -61,18 +61,19 @@ impl Controller {
             }
         };
 
+        let dav_hdrs = node.dav_header();
         let ctrl = Self { node, user, req };
 
         match method.as_str() {
             "OPTIONS" => Ok(Response::builder()
                 .status(200)
-                .header("DAV", "1")
+                .header("DAV", dav_hdrs)
                 .header("Allow", "HEAD,GET,PUT,OPTIONS,DELETE,PROPFIND,PROPPATCH,MKCOL,COPY,MOVE,LOCK,UNLOCK,MKCALENDAR,REPORT")
                 .body(codec::text_body(""))?),
             "HEAD" => {
-                tracing::warn!("HEAD not correctly implemented");
+                tracing::warn!("HEAD might not correctly implemented: should return ETags & co");
                 Ok(Response::builder()
-                    .status(404)
+                    .status(200)
                     .body(codec::text_body(""))?)
             },
             "GET" => ctrl.get().await,
@@ -184,7 +185,7 @@ impl Controller {
                     },
                 };
                 extension = Some(realization::Multistatus::Sync(sync::Multistatus {
-                    sync_token: sync::SyncToken(new_token.to_string()),
+                    sync_token: sync::SyncToken(format!("{}{}", BASE_TOKEN_URI, new_token)),
                 }));
             }
             _ => {
@@ -348,11 +349,14 @@ impl Controller {
         }
 
         // Build response
-        dav::Multistatus::<All> {
+        let multistatus = dav::Multistatus::<All> {
             responses,
             responsedescription: None,
             extension,
-        }
+        };
+
+        tracing::debug!(multistatus=?multistatus, "multistatus response");
+        multistatus
     }
 }
 

aero-proto/src/dav/mod.rs

@@ -98,7 +98,7 @@ impl Server {
             let conn = tokio::spawn(async move {
                 //@FIXME should create a generic "public web" server on which "routers" could be
                 //abitrarily bound
-                //@FIXME replace with a handler supporting http2 and TLS
+                //@FIXME replace with a handler supporting http2
 
                 match http::Builder::new()
                     .serve_connection(
@@ -106,8 +106,9 @@ impl Server {
                         service_fn(|req: Request<hyper::body::Incoming>| {
                             let login = login.clone();
                             tracing::info!("{:?} {:?}", req.method(), req.uri());
+                            tracing::debug!(req=?req, "full request");
                             async {
-                                match middleware::auth(login, req, |user, request| {
+                                let response = match middleware::auth(login, req, |user, request| {
                                     async { Controller::route(user, request).await }.boxed()
                                 })
                                 .await
@@ -119,7 +120,9 @@ impl Server {
                                             .status(500)
                                             .body(codec::text_body("Internal error"))
                                     }
-                                }
+                                };
+                                tracing::debug!(resp=?response, "full response");
+                                response
                             }
                         }),
                     )

aero-proto/src/dav/node.rs

@@ -40,7 +40,9 @@ pub(crate) trait DavNode: Send {
     fn supported_properties(&self, user: &ArcUser) -> dav::PropName<All>;
     /// Get the values for the given properties
     fn properties(&self, user: &ArcUser, prop: dav::PropName<All>) -> PropertyStream<'static>;
-    //fn properties(&self, user: &ArcUser, prop: dav::PropName<All>) -> Vec<dav::AnyProperty<All>>;
+    /// Get the value of the DAV header to return
+    fn dav_header(&self) -> String;
+
     /// Put an element (create or update)
     fn put<'a>(
         &'a self,

aero-proto/src/dav/resource.rs

@@ -21,6 +21,16 @@ use aero_dav::versioningtypes as vers;
 use super::node::PropertyStream;
 use crate::dav::node::{Content, DavNode, PutPolicy};
 
+/// Why "https://aerogramme.0"?
+/// Because tokens must be valid URI.
+/// And numeric TLD are ~mostly valid in URI (check the .42 TLD experience)
+/// and at the same time, they are not used sold by the ICANN and there is no plan to use them.
+/// So I am sure that the URL remains invalid, avoiding leaking requests to an hardcoded URL in the
+/// future.
+/// The best option would be to make it configurable ofc, so someone can put a domain name
+/// that they control, it would probably improve compatibility (maybe some WebDAV spec tells us
+/// how to handle/resolve this URI but I am not aware of that...). But that's not the plan for
+/// now. So here we are: https://aerogramme.0.
 pub const BASE_TOKEN_URI: &str = "https://aerogramme.0/sync/";
 
 #[derive(Clone)]
@@ -129,6 +139,10 @@ impl DavNode for RootNode {
     > {
         async { Err(std::io::Error::from(std::io::ErrorKind::Unsupported)) }.boxed()
     }
+
+    fn dav_header(&self) -> String {
+        "1".into()
+    }
 }
 
 #[derive(Clone)]
@@ -250,6 +264,10 @@ impl DavNode for HomeNode {
     > {
         async { Err(std::io::Error::from(std::io::ErrorKind::Unsupported)) }.boxed()
     }
+
+    fn dav_header(&self) -> String {
+        "1, access-control, calendar-access".into()
+    }
 }
 
 #[derive(Clone)]
@@ -383,6 +401,10 @@ impl DavNode for CalendarListNode {
     > {
         async { Err(std::io::Error::from(std::io::ErrorKind::Unsupported)) }.boxed()
     }
+
+    fn dav_header(&self) -> String {
+        "1, access-control, calendar-access".into()
+    }
 }
 
 #[derive(Clone)]
@@ -575,7 +597,31 @@ impl DavNode for CalendarNode {
         let col = self.col.clone();
         let calname = self.calname.clone();
         async move {
-            let sync_token = sync_token.unwrap();
+            let sync_token = match sync_token {
+                Some(v) => v,
+                None => {
+                    let token = col
+                        .token()
+                        .await
+                        .or(Err(std::io::Error::from(std::io::ErrorKind::Interrupted)))?;
+                    let ok_nodes = col
+                        .dag()
+                        .await
+                        .idx_by_filename
+                        .iter()
+                        .map(|(filename, blob_id)| {
+                            Box::new(EventNode {
+                                col: col.clone(),
+                                calname: calname.clone(),
+                                filename: filename.to_string(),
+                                blob_id: *blob_id,
+                            }) as Box<dyn DavNode>
+                        })
+                        .collect();
+
+                    return Ok((token, ok_nodes, vec![]));
+                }
+            };
             let (new_token, listed_changes) = match col.diff(sync_token).await {
                 Ok(v) => v,
                 Err(e) => {
@@ -607,6 +653,9 @@ impl DavNode for CalendarNode {
         }
         .boxed()
     }
+    fn dav_header(&self) -> String {
+        "1, access-control, calendar-access".into()
+    }
 }
 
 #[derive(Clone)]
@@ -838,6 +887,10 @@ impl DavNode for EventNode {
     > {
         async { Err(std::io::Error::from(std::io::ErrorKind::Unsupported)) }.boxed()
     }
+
+    fn dav_header(&self) -> String {
+        "1, access-control".into()
+    }
 }
 
 #[derive(Clone)]
@@ -939,4 +992,8 @@ impl DavNode for CreateEventNode {
     > {
         async { Err(std::io::Error::from(std::io::ErrorKind::Unsupported)) }.boxed()
     }
+
+    fn dav_header(&self) -> String {
+        "1, access-control".into()
+    }
 }

aerogramme/tests/behavior.rs

@@ -6,7 +6,7 @@ use crate::common::fragments::*;
 
 fn main() {
     // IMAP
-    /*rfc3501_imap4rev1_base();
+    rfc3501_imap4rev1_base();
     rfc6851_imapext_move();
     rfc4551_imapext_condstore();
     rfc2177_imapext_idle();
@@ -14,7 +14,7 @@ fn main() {
     rfc3691_imapext_unselect();
     rfc7888_imapext_literal();
     rfc4315_imapext_uidplus();
-    rfc5819_imapext_liststatus();*/
+    rfc5819_imapext_liststatus();
 
     // WebDAV
     rfc4918_webdav_core();
@@ -370,7 +370,7 @@ use aero_dav::synctypes as sync;
 use aero_dav::types as dav;
 use aero_dav::versioningtypes as vers;
 
-use crate::common::dav_deserialize;
+use crate::common::{dav_deserialize, dav_serialize};
 
 fn rfc4918_webdav_core() {
     println!("🧪 rfc4918_webdav_core");
@@ -435,6 +435,7 @@ fn rfc4918_webdav_core() {
         assert!(root_success.prop.0.iter().find(|p| matches!(p, dav::AnyProperty::Value(dav::Property::GetContentType(_)))).is_none());
         assert!(root_not_found.prop.0.iter().find(|p| matches!(p, dav::AnyProperty::Request(dav::PropertyRequest::GetContentLength))).is_some());
 
+        // -- HIERARCHY EXPLORATION WITH THE DEPTH: X HEADER FIELD --
         // depth 1 / -> /alice/
         let body = http.request(reqwest::Method::from_bytes(b"PROPFIND")?, "http://localhost:8087").header("Depth", "1").send()?.text()?;
         let multistatus = dav_deserialize::<dav::Multistatus<All>>(&body);
@@ -470,7 +471,7 @@ fn rfc4918_webdav_core() {
         let multistatus = dav_deserialize::<dav::Multistatus<All>>(&body);
         assert_eq!(multistatus.responses.len(), 1);
 
-        // --- PUT ---
+        // --- PUT (add objets) ---
         // first object
         let resp = http.put("http://localhost:8087/alice/calendar/Personal/rfc2.ics").header("If-None-Match", "*").body(ICAL_RFC2).send()?;
         let obj1_etag = resp.headers().get("etag").expect("etag must be set");
@@ -496,14 +497,14 @@ fn rfc4918_webdav_core() {
         let resp = http.put("http://localhost:8087/alice/calendar/Personal/rfc2.ics").header("If-Match", obj1_etag).body(ICAL_RFC1).send()?;
         assert_eq!(resp.status(), 201);
 
-        // --- GET ---
+        // --- GET (fetch objects) ---
         let body = http.get("http://localhost:8087/alice/calendar/Personal/rfc2.ics").send()?.text()?;
         assert_eq!(body.as_bytes(), ICAL_RFC1);
 
         let body = http.get("http://localhost:8087/alice/calendar/Personal/rfc3.ics").send()?.text()?;
         assert_eq!(body.as_bytes(), ICAL_RFC3);
 
-        // --- DELETE ---
+        // --- DELETE (delete objects) ---
         // delete 1st object
         let resp = http.delete("http://localhost:8087/alice/calendar/Personal/rfc2.ics").send()?;
         assert_eq!(resp.status(), 204);
@@ -528,7 +529,7 @@ fn rfc4918_webdav_core() {
 fn rfc5397_webdav_principal() {
     println!("🧪 rfc5397_webdav_principal");
     common::aerogramme_provider_daemon_dev(|_imap, _lmtp, http| {
-        // Find principal
+        // -- AUTODISCOVERY: FIND "PRINCIPAL" AS DEFINED IN WEBDAV ACL (~USER'S HOME) --
         let propfind_req = r#"<?xml version="1.0" encoding="utf-8" ?><propfind xmlns="DAV:"><prop><current-user-principal/></prop></propfind>"#;
         let body = http.request(reqwest::Method::from_bytes(b"PROPFIND")?, "http://localhost:8087").body(propfind_req).send()?.text()?;
         let multistatus = dav_deserialize::<dav::Multistatus<All>>(&body);
@@ -1017,7 +1018,8 @@ fn rfc4791_webdav_caldav() {
 fn rfc6578_webdav_sync() {
     println!("🧪 rfc6578_webdav_sync");
     common::aerogramme_provider_daemon_dev(|_imap, _lmtp, http| {
-        // propname on a calendar node must return <sync-token/> + <supported-report-set/> (2nd element is theoretically from versioning)
+        // -- PROPFIND --
+        // propname must return sync-token & supported-report-set (from webdav versioning)
         let propfind_req = r#"<?xml version="1.0" encoding="utf-8" ?><propfind xmlns="DAV:"><propname/></propfind>"#;
         let body = http.request(reqwest::Method::from_bytes(b"PROPFIND")?, "http://localhost:8087/alice/calendar/Personal/").body(propfind_req).send()?.text()?;
         let multistatus = dav_deserialize::<dav::Multistatus<All>>(&body);
@@ -1110,6 +1112,180 @@ fn rfc6578_webdav_sync() {
         assert!(init_sync_token != del_sync_token);
         assert!(rfc1_sync_token != del_sync_token);
 
+        // -- TEST SYNC CUSTOM REPORT: SYNC-COLLECTION --
+        // 3.8.  Example: Initial DAV:sync-collection Report
+        // Part 1: check the empty case
+        let sync_query = r#"<?xml version="1.0" encoding="utf-8" ?>
+            <D:sync-collection xmlns:D="DAV:">
+                <D:sync-token/>
+                <D:sync-level>1</D:sync-level>
+                <D:prop>
+                    <D:getetag/>
+                </D:prop>
+            </D:sync-collection>
+        "#;
+        let resp = http
+            .request(
+                reqwest::Method::from_bytes(b"REPORT")?,
+                "http://localhost:8087/alice/calendar/Personal/",
+            )
+            .body(sync_query)
+            .send()?;
+        assert_eq!(resp.status(), 207);
+        let multistatus = dav_deserialize::<dav::Multistatus<All>>(&resp.text()?);
+        assert_eq!(multistatus.responses.len(), 0);
+        let empty_token = match &multistatus.extension {
+            Some(realization::Multistatus::Sync(sync::Multistatus { sync_token: sync::SyncToken(x) } )) => x,
+            _ => anyhow::bail!("wrong content"),
+        };
+
+        // Part 2: check with one file
+        let resp = http
+            .put("http://localhost:8087/alice/calendar/Personal/rfc1.ics")
+            .header("If-None-Match", "*")
+            .body(ICAL_RFC1)
+            .send()?;
+        assert_eq!(resp.status(), 201);
+
+        let resp = http
+            .request(
+                reqwest::Method::from_bytes(b"REPORT")?,
+                "http://localhost:8087/alice/calendar/Personal/",
+            )
+            .body(sync_query)
+            .send()?;
+        assert_eq!(resp.status(), 207);
+        let multistatus = dav_deserialize::<dav::Multistatus<All>>(&resp.text()?);
+        assert_eq!(multistatus.responses.len(), 1);
+        let initial_one_file_token = match &multistatus.extension {
+            Some(realization::Multistatus::Sync(sync::Multistatus { sync_token: sync::SyncToken(x) } )) => x,
+            _ => anyhow::bail!("wrong content"),
+        };
+        assert!(empty_token != initial_one_file_token);
+
+        // 3.9.  Example: DAV:sync-collection Report with Token
+        // Part 1: nothing changed, response must be empty
+        let sync_query = |token: &str| vers::Report::<realization::All>::Extension(realization::ReportType::Sync(sync::SyncCollection {
+            sync_token: sync::SyncTokenRequest::IncrementalSync(token.into()),
+            sync_level: sync::SyncLevel::One,
+            limit: None,
+            prop: dav::PropName(vec![
+                dav::PropertyRequest::GetEtag,
+            ]),
+        }));
+        let resp = http
+            .request(
+                reqwest::Method::from_bytes(b"REPORT")?,
+                "http://localhost:8087/alice/calendar/Personal/",
+            )
+            .body(dav_serialize(&sync_query(initial_one_file_token)))
+            .send()?;
+        assert_eq!(resp.status(), 207);
+        let multistatus = dav_deserialize::<dav::Multistatus<All>>(&resp.text()?);
+        assert_eq!(multistatus.responses.len(), 0);
+        let no_change = match &multistatus.extension {
+            Some(realization::Multistatus::Sync(sync::Multistatus { sync_token: sync::SyncToken(x) } )) => x,
+            _ => anyhow::bail!("wrong content"),
+        };
+        assert_eq!(initial_one_file_token, no_change);
+
+        // Part 2: add a new node (rfc2) + remove a node (rfc1)
+        // add rfc2
+        let resp = http
+            .put("http://localhost:8087/alice/calendar/Personal/rfc2.ics")
+            .header("If-None-Match", "*")
+            .body(ICAL_RFC2)
+            .send()?;
+        assert_eq!(resp.status(), 201);
+
+        // delete rfc1
+        let resp = http.delete("http://localhost:8087/alice/calendar/Personal/rfc1.ics").send()?;
+        assert_eq!(resp.status(), 204);
+
+        // call REPORT <sync-collection>
+        let resp = http
+            .request(
+                reqwest::Method::from_bytes(b"REPORT")?,
+                "http://localhost:8087/alice/calendar/Personal/",
+            )
+            .body(dav_serialize(&sync_query(initial_one_file_token)))
+            .send()?;
+        assert_eq!(resp.status(), 207);
+        let multistatus = dav_deserialize::<dav::Multistatus<All>>(&resp.text()?);
+        assert_eq!(multistatus.responses.len(), 2);
+        let token_addrm = match &multistatus.extension {
+            Some(realization::Multistatus::Sync(sync::Multistatus { sync_token: sync::SyncToken(x) } )) => x,
+            _ => anyhow::bail!("wrong content"),
+        };
+        assert!(initial_one_file_token != token_addrm);
+
+        // Part 3: remove a node (rfc2) and add it again with new content
+        // delete rfc2
+        let resp = http.delete("http://localhost:8087/alice/calendar/Personal/rfc2.ics").send()?;
+        assert_eq!(resp.status(), 204);
+
+        // add rfc2 with ICAL_RFC3 content
+        let resp = http
+            .put("http://localhost:8087/alice/calendar/Personal/rfc2.ics")
+            .header("If-None-Match", "*")
+            .body(ICAL_RFC3)
+            .send()?;
+        let rfc2_etag = resp.headers().get("etag").expect("etag must be set");
+        assert_eq!(resp.status(), 201);
+
+        // call REPORT <sync-collection>
+        let resp = http
+            .request(
+                reqwest::Method::from_bytes(b"REPORT")?,
+                "http://localhost:8087/alice/calendar/Personal/",
+            )
+            .body(dav_serialize(&sync_query(token_addrm)))
+            .send()?;
+        assert_eq!(resp.status(), 207);
+        let multistatus = dav_deserialize::<dav::Multistatus<All>>(&resp.text()?);
+        assert_eq!(multistatus.responses.len(), 1);
+        let token_addrm_same = match &multistatus.extension {
+            Some(realization::Multistatus::Sync(sync::Multistatus { sync_token: sync::SyncToken(x) } )) => x,
+            _ => anyhow::bail!("wrong content"),
+        };
+        assert!(token_addrm_same != token_addrm);
+
+        // Part 4: overwrite an event (rfc1) with new content
+        let resp = http
+            .put("http://localhost:8087/alice/calendar/Personal/rfc1.ics")
+            .header("If-Match", rfc2_etag)
+            .body(ICAL_RFC4)
+            .send()?;
+        assert_eq!(resp.status(), 201);
+
+        // call REPORT <sync-collection>
+        let resp = http
+            .request(
+                reqwest::Method::from_bytes(b"REPORT")?,
+                "http://localhost:8087/alice/calendar/Personal/",
+            )
+            .body(dav_serialize(&sync_query(token_addrm_same)))
+            .send()?;
+        assert_eq!(resp.status(), 207);
+        let multistatus = dav_deserialize::<dav::Multistatus<All>>(&resp.text()?);
+        assert_eq!(multistatus.responses.len(), 1);
+        let token_addrm_same = match &multistatus.extension {
+            Some(realization::Multistatus::Sync(sync::Multistatus { sync_token: sync::SyncToken(x) } )) => x,
+            _ => anyhow::bail!("wrong content"),
+        };
+        assert!(token_addrm_same != token_addrm);
+
+        // Unknown token must return 410 GONE.
+        // Token can be forgotten as we garbage collect the DAG.
+        let resp = http
+            .request(
+                reqwest::Method::from_bytes(b"REPORT")?,
+                "http://localhost:8087/alice/calendar/Personal/",
+            )
+            .body(dav_serialize(&sync_query("https://aerogramme.0/sync/000000000000000000000000000000000000000000000000")))
+            .send()?;
+        assert_eq!(resp.status(), 410);
+
         Ok(())
     })
     .expect("test fully run")

aerogramme/tests/common/mod.rs

@@ -108,7 +108,8 @@ pub fn read_first_u32(inp: &str) -> Result<u32> {
         .parse::<u32>()?)
 }
 
-use aero_dav::xml::{Node, Reader};
+use aero_dav::xml::{Node, Reader, Writer};
+use tokio::io::AsyncWriteExt;
 pub fn dav_deserialize<T: Node<T>>(src: &str) -> T {
     futures::executor::block_on(async {
         let mut rdr = Reader::new(quick_xml::NsReader::from_reader(src.as_bytes()))
@@ -117,3 +118,19 @@ pub fn dav_deserialize<T: Node<T>>(src: &str) -> T {
         rdr.find().await.expect("parse XML")
     })
 }
+pub fn dav_serialize<T: Node<T>>(src: &T) -> String {
+    futures::executor::block_on(async {
+        let mut buffer = Vec::new();
+        let mut tokio_buffer = tokio::io::BufWriter::new(&mut buffer);
+        let q = quick_xml::writer::Writer::new_with_indent(&mut tokio_buffer, b' ', 4);
+        let ns_to_apply = vec![
+            ("xmlns:D".into(), "DAV:".into()),
+            ("xmlns:C".into(), "urn:ietf:params:xml:ns:caldav".into()),
+        ];
+        let mut writer = Writer { q, ns_to_apply };
+
+        src.qwrite(&mut writer).await.expect("xml serialization");
+        tokio_buffer.flush().await.expect("tokio buffer flush");
+        std::str::from_utf8(buffer.as_slice()).unwrap().into()
+    })
+}