From ec061022e0b4bce8d5993d35a9c4ed494c44ecdd Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Tue, 25 Jul 2023 19:08:48 +0200 Subject: finalize eml-codec integration --- Cargo.lock | 4 +- Cargo.toml | 3 +- README.md | 250 +++++++----------------------------- flake.lock | 36 +++--- flake.nix | 11 +- src/imap/mailbox_view.rs | 126 +++++++++--------- src/login/mod.rs | 2 +- src/mail/mailbox.rs | 6 +- tests/parsing-crates/mail_parser.rs | 73 ----------- tests/parsing-crates/main.rs | 1 - 10 files changed, 142 insertions(+), 370 deletions(-) delete mode 100644 tests/parsing-crates/mail_parser.rs delete mode 100644 tests/parsing-crates/main.rs diff --git a/Cargo.lock b/Cargo.lock index c795d01..5da7547 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -923,7 +923,9 @@ checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" [[package]] name = "eml-codec" -version = "0.1.0" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac20cff537caf72385ffa5d9353ae63cb6c283a53665569408f040b8db36c90d" dependencies = [ "base64 0.21.2", "chrono", diff --git a/Cargo.toml b/Cargo.toml index cf0359b..14143da 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,8 +14,7 @@ backtrace = "0.3" base64 = "0.13" clap = { version = "3.1.18", features = ["derive", "env"] } duplexify = "1.1.0" -#eml-codec = { path = "../eml-codec" } -eml-codec = "0.1.0" +eml-codec = "0.1.1" hex = "0.4" futures = "0.3" im = "15" diff --git a/README.md b/README.md index 7f5254d..4d7d947 100644 --- a/README.md +++ b/README.md @@ -1,209 +1,45 @@ # Aerogramme - Encrypted e-mail storage over Garage -## Nix builds - -you can cross compile static binaries with: - -```bash -nix build -L .#packages.x86_64-unknown-linux-musl.default # linux/amd64 -nix build -L .#packages.aarch64-unknown-linux-musl.default # linux/arm64 -nix build -L .#packages.armv6l-unknown-linux-musleabihf.default # linux/arm -``` - -## Usage - -Start by running: - -``` -$ cargo run --bin main -- first-login --region garage --k2v-endpoint http://127.0.0.1:3904 --s3-endpoint http://127.0.0.1:3900 --aws-access-key-id GK... --aws-secret-access-key c0ffee... --bucket mailrage-quentin --user-secret poupou -Please enter your password for key decryption. -If you are using LDAP login, this must be your LDAP password. -If you are using the static login provider, enter any password, and this will also become your password for local IMAP access. -Enter password: -Confirm password: - -Cryptographic key setup is complete. - -If you are using the static login provider, add the following section to your .toml configuration file: - -[login_static.users.] -password = "$argon2id$v=19$m=4096,t=3,p=1$..." -aws_access_key_id = "GK..." -aws_secret_access_key = "c0ffee..." -``` - -Next create the config file `aerogramme.toml`: - -``` -s3_endpoint = "http://127.0.0.1:3900" -k2v_endpoint = "http://127.0.0.1:3904" -aws_region = "garage" - -[login_static] -default_bucket = "mailrage" -[login_static.users.quentin] -bucket = "mailrage-quentin" -user_secret = "poupou" -alternate_user_secrets = [] -password = "$argon2id$v=19$m=4096,t=3,p=1$..." -aws_access_key_id = "GK..." -aws_secret_access_key = "c0ffee..." -``` - -You can dump your keys with: - -``` -$ cargo run --bin main -- show-keys --region garage --k2v-endpoint http://127.0.0.1:3904 --s3-endpoint http://127.0.0.1:3900 --aws-access-key-id GK... --aws-secret-access-key c0ffee... --bucket mailrage-quentin --user-secret poupou -Enter key decryption password: -master_key = "..." -secret_key = "..." -``` - -Run a test instance with: - -``` -$ cargo run --bin main -- server ----- MAILBOX STATE ---- -UIDVALIDITY 1 -UIDNEXT 2 -INTERNALSEQ 2 -1 c3d4524f557f19108480063f3216afa20000000000000000 \Unseen - ----- MAILBOX STATE ---- -UIDVALIDITY 1 -UIDNEXT 3 -INTERNALSEQ 3 -1 c3d4524f557f19108480063f3216afa20000000000000000 \Unseen -2 6a1ab4d87af3d424a3a8f8720c4db3b60000000000000000 \Unseen -``` - - -## Bayou storage module - -Checkpoints are stored in S3 at `/checkpoint/`. Example: - -``` -348 TestMailbox/checkpoint/00000180d77400dc126b16aac546b769 -369 TestMailbox/checkpoint/00000180d776e509b68fdc5c376d0abc -357 TestMailbox/checkpoint/00000180d77a7fe68f4f76e3b45aa751 -``` - -Operations are stored in K2V at PK ``, SK ``. Example: - -``` -TestMailbox 00000180d77400dc126b16aac546b769 RcIsESv7WrjMuHwyI/dvCnkIfy6op5Tiylf0WSnn94aMS2uagl7YeMBwdv09TiSXBpu5nJ5e/9QFSfuEI/NqKrdQkX54MOsnaIGhRb0oqUG3KNaar3BiVSvYvXuzYhk4ii+TUS2Eyd6fCCaNVNM5 -TestMailbox 00000180d775f27f5542a13fc21c665e RrTSOup/zO1Ei+QrjBcDLt4vvFSY+WJPBodwY64wy2ftW+Oh3VSArvlO4SAEPmdsx1gt0HPBZYR/OkVWsZpmix1ZLFUmvdib+rjNkorHQW1p+oLVK8tolGrqk4SRwl88cqu466T4vBEpDu7tRbH0 -TestMailbox 00000180d775f292b3c8da00718389b4 VAwd8SRycIwsipZW5AcSG+EIYZVWn/Uj/TADbWhb4x5LVMceiRBHWVquY08RgT/lJKdhIcUqBA15bVG3klIg8tLsWJVG784NbsZwdGRczWmngcA= -TestMailbox 00000180d775f29d24842cf375d679e0 /FbXtEwm/bijtvOdqM1XFvKUalQFAOPHp+vF9jZThZn/viY5a6W1PyHeI8kTusF6EsVPAwPHpQyjIv/ghskC0f+zUEsSUhDwQANdwLNqDLAvTA== -TestMailbox 00000180d7768ab1dc01ff504e887c62 W/fF0WitpxJ05yHeOv96BlpGymT1kVOjkIW00t9e6UE7mxkvNflu9cZSCd8PDJd2ymC0sC9bLVFAXKmNZsmCFEEHMQSyrX61qTYo4KFCZMp5zm6fXubaYuurrzjXzfUP/R7kBvICFZlF0daf0SwX -TestMailbox 00000180d7768aba629c7ad6adf25228 IPzYGNsSepCX2AEnee/1Eas9a3c5esPSmrNkvaj4XcFb6Ft2KC8N6ubUR3wB+K0oYCTQym6nhHG5dlAxf6NRu7Rk8YtBTBmSqtGqd6kMZ3bU5b8= -TestMailbox 00000180d7768ac1870cda61784114d4 aaLiaWxfx1mxh6aoKE3xUUfZWhivZ/K7ixabflFDW7FO/qbpvCaa+Y6w4lQemTy6m+leAhXGN+Dbyv2qP20yJ9O4oJF5d3Lz5Iv5uF18OxhVZzw= -TestMailbox 00000180d776e4fb294ccdab2612b406 EtUPrLgEeOyab2QRnSie4I3Me9dDh10UdwWnUKdGa/8ezMJDtiy7XlW+tUfJdqtu6Vj7nduT0emDOXbBZsNwlcmzgYNwuNu3I9AfhZTFWtwLgB+wnAgB/jim82DDrJfLia8kB2eA2ao5jfJ3uMSZ -TestMailbox 00000180d776e501528546d340490291 Lz4Z9wCTk1lZ86lL01urhAan4oHcr1NBqdRe+CDpA51D9IncA5+Fhc8I6knUIh2qQ5/woWgISLAVwzSS+0+TxrYoqxf5FumIQtUJfwDER5La3n0= -TestMailbox 00000180d776e509b68fdc5c376d0abc RUGE2xB3fFX/wRH/p2fHIUa+rMaXSRd7fY9zglw0pRfVPqJfpniOjAe4GHIwGlwbwjtFOwS5a+Q7yr0Wez6QwD+ohhqRFKpbjcFcN7VfMyVAf+k= -TestMailbox 00000180d7784b987a8ad8106dc400c9 K+0LVEtBbTnWNS67jy9DtTvQyd5arovduvu490tLOE2TzVhuVoF4pfvTMTN12bH3KwEAHeDfuwKkKJFqldOywouTYPzEjZFkJzyagHrkl6dfnE5CqmlDv+Vc5TOQRskxjW+wQiZdjU8wGiBiBGYh -TestMailbox 00000180d7784bede69ac3cff2c6b724 XMFY3+b1r1//uolVz80JSI3g/84XCk3Tm7/S0BFv+Qe/Xv3/poLrOvAKEe+GzD2s22j8p/T2RXR/JSZckzgjEZeO0wbPDXVQd94di2Pff7jxAH8= -TestMailbox 00000180d7784bffe2595abe7ed81858 QQZhF+7wSHfikoAp93a+UY/XDIX7TVnnVYOtmQ2XHnDKA2F6snRJCPbYBO4IRHCRfVrjDGi32c41it2C3Mu5PBepabxapsW1rfIV3rlX2lkKHtI= -TestMailbox 00000180d77a7fb3f01dbb147c20cf7f IHOlOa1JI11RUKVvQUq3HQPxiRr4UCeE+pHmL8DtNMkOh62V4spuP0VvvQTJCQcPQ1EQR/QcxZ3s7uHLkrZAHF30BkpUkGqsLBWpnyug/puhdiixWsMyLLb6G90zFjiComUwptnDc/CCXtGEHdSW -TestMailbox 00000180d77a7fbb54b100f521ceb347 Ze4KyyTCgrYbZlXlJSY5hNob8sMXvBAmwIx2cADbX5P0M1IHXwXfloEzvvd6WYOtatFC2GnDSrmQ6RdCfeZ3WV9TZilqa0Fv0XEg48sVyVCcguw= -TestMailbox 00000180d77a7fe68f4f76e3b45aa751 cJJVvvRzTVNKUaIHPCCDY2uY7/HlmkxGgo3ozWBlBSRDeBqU65zgZD3QIPCxa6xaqB/Gc0bQ9BGzfU0cvVmO5jgNeeDnbqqs3oeA2jml/Qv2YO9upApfNQtDT1GiwJ8vrgaIow== -TestMailbox 00000180d8e513d3ea58c679a13178ac Ce5su2YOxNmTzk2dK8SX8V/Uue5uAC7oklEjhesY9wCMqGphhOkdWjzCqq0xOzcb/ZzzZ58t+mTksNSYIU4kddHIHBFPgqIwKthVk2mlUdqYiN/Y2vEGqv+YmtKY+GST/7Ee87ZHpU/5sv0GoXxT -TestMailbox 00000180d8e5145a23f8faee86283900 sp3D8xFZcM9icNlDJXIUDJb3mo6VGD9f1aDHD+4RbPdx6mTYF+qNTsPHKCxHHxT/9NfNe8XPg2+8xYRtm7SXfgERZBDB8ye+Xt3fM1k+wbL6RsaJmDHVECeXeL5KHuITzpI22A== -TestMailbox 00000180d8e51465c38f0585f9bb760e FF0VId2O/bBNzYD5ABWReMs5hHoHwynOoJRKj9vyaUMZ3JykInFmvvRgtCbJBDjTQPwPU8apphKQfwuicO76H7GtZqH009Cbv5l8ZTRJKrmzOQmtjzBQc2eGEUMPfbml5t0GCg== -``` - -The timestamp of a checkpoint corresponds to the timestamp of the first operation NOT included in the checkpoint. -In other words, to reconstruct the final state: - -- find timestamp `` of last checkpoint -- load checkpoint `` -- load and apply all operations starting from ``, included - -## UID index - -The UID index is an application of the Bayou storage module -used to assign UID numbers to e-mails. -See document we sent to NGI for properties on UIDVALIDITY. - -## Cryptography; key management - -Keys that are used: - -- master secret key (for indexes) -- curve25519 public/private key pair (for incoming mail) - -Keys that are stored in K2V under PK `keys`: - -- `public`: the public curve25519 key (plain text) -- `salt`: the 32-byte salt `S` used to calculate digests that index keys below -- if a password is used, `password:`: - - a 32-byte salt `Skey` - - followed a secret box - - that is encrypted with a strong argon2 digest of the password (using the salt `Skey`) and a user secret (see below) - - that contains the master secret key and the curve25519 private key - -User secret: an additionnal secret that is added to the password when deriving the encryption key for the secret box. -This additionnal secret should not be stored in K2V/S3, so that just knowing a user's password isn't enough to be able -to decrypt their mailbox (supposing the attacker has a dump of their K2V/S3 bucket). -This user secret should typically be stored in the LDAP database or just in the configuration file when using -the static login provider. - -Operations: - -- **Initialize**(`user_secret`, `password`): - - if `"salt"` or `"public"` already exist, BAIL - - generate salt `S` (32 random bytes) - - generate `public`, `private` (curve25519 keypair) - - generate `master` (secretbox secret key) - - calculate `digest = argon2_S(password)` - - generate salt `Skey` (32 random bytes) - - calculate `key = argon2_Skey(user_secret + password)` - - serialize `box_contents = (private, master)` - - seal box `blob = seal_key(box_contents)` - - write `S` at `"salt"` - - write `concat(Skey, blob)` at `"password:{hex(digest[..16])}"` - - write `public` at `"public"` - -- **InitializeWithoutPassword**(`private`, `master`): - - if `"salt"` or `"public"` already exist, BAIL - - generate salt `S` (32 random bytes) - - write `S` at `"salt"` - - calculate `public` the public key associated with `private` - - write `public` at `"public"` - -- **Open**(`user_secret`, `password`): - - load `S = read("salt")` - - calculate `digest = argon2_S(password)` - - load `blob = read("password:{hex(digest[..16])}") - - set `Skey = blob[..32]` - - calculate `key = argon2_Skey(user_secret + password)` - - open secret box `box_contents = open_key(blob[32..])` - - retrieve `master` and `private` from `box_contents` - - retrieve `public = read("public")` - -- **OpenWithoutPassword**(`private`, `master`): - - load `public = read("public")` - - check that `public` is the correct public key associated with `private` - -- **AddPassword**(`user_secret`, `existing_password`, `new_password`): - - load `S = read("salt")` - - calculate `digest = argon2_S(existing_password)` - - load `blob = read("existing_password:{hex(digest[..16])}") - - set `Skey = blob[..32]` - - calculate `key = argon2_Skey(user_secret + existing_password)` - - open secret box `box_contents = open_key(blob[32..])` - - retrieve `master` and `private` from `box_contents` - - - calculate `digest_new = argon2_S(new_password)` - - generate salt `Skeynew` (32 random bytes) - - calculate `key_new = argon2_Skeynew(user_secret + new_password)` - - serialize `box_contents_new = (private, master)` - - seal box `blob_new = seal_key_new(box_contents_new)` - - write `concat(Skeynew, blob_new)` at `"new_password:{hex(digest_new[..16])}"` - -- **RemovePassword**(`password`): - - load `S = read("salt")` - - calculate `digest = argon2_S(existing_password)` - - check that `"password:{hex(digest[..16])}"` exists - - check that other passwords exist ?? (or not) - - delete `"password:{hex(digest[..16])}"` +⚠️ **TECHNOLOGICAL PREVIEW, THIS SERVER IS NOT READY FOR PRODUCTION OR EVEN BETA TESTING** + +![Aerogramme logo](https://aerogramme.deuxfleurs.fr/logo/aerogramme-blue-hz.svg) + +A resilient & standards-compliant open-source IMAP server with built-in encryption + +## Quickly jump to our website! + +[![Download](https://aerogramme.deuxfleurs.fr/images/download.png)](https://aerogramme.deuxfleurs.fr/download/) [![Getting Started](https://aerogramme.deuxfleurs.fr/images/getting-started.png)}(https://aerogramme.deuxfleurs.fr/documentation/quick-start/) + +[RFC Coverage](https://aerogramme.deuxfleurs.fr/documentation/reference/rfc/) - [Design overview](https://aerogramme.deuxfleurs.fr/documentation/design/overview/) - [Mailbox Datastructure](https://aerogramme.deuxfleurs.fr/documentation/design/mailbox/) - [Mailbox Mutation Log](https://aerogramme.deuxfleurs.fr/documentation/design/log/). + +## Roadmap + + - ✅ 0.1 Better emails parsing (july '23, see [eml-codec](https://git.deuxfleurs.fr/Deuxfleurs/eml-codec)). + - ⌛0.2 Support of IMAP4rev1. (~september '23). + - ⌛0.3 Subset of IMAP4rev2. (~december '23). + - ⌛0.4 CalDAV support. (~february '24). + - ⌛0.5 CardDAV support. + +## Sponsors and funding + +[Aerogramme project](https://nlnet.nl/project/Aerogramme/) is funded through the NGI Assure Fund, a fund established by NLnet with financial support from the European Commission's Next Generation Internet programme, under the aegis of DG Communications Networks, Content and Technology under grant agreement No 957073. + +![NLnet logo](https://aerogramme.deuxfleurs.fr/images/nlnet.svg) + +## License + + EUROPEAN UNION PUBLIC LICENCE v. 1.2 + EUPL © the European Union 2007, 2016 + +This European Union Public Licence (the ‘EUPL’) applies to the Work (as defined +below) which is provided under the terms of this Licence. Any use of the Work, +other than as authorised under this Licence is prohibited (to the extent such +use is covered by a right of the copyright holder of the Work). + +The Work is provided under the terms of this Licence when the Licensor (as +defined below) has placed the following notice immediately following the +copyright notice for the Work: + + Licensed under the EUPL + +or has expressed by any other means his willingness to license under the EUPL. diff --git a/flake.lock b/flake.lock index 39f5848..4ada7c3 100644 --- a/flake.lock +++ b/flake.lock @@ -28,11 +28,11 @@ "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1682038649, - "narHash": "sha256-HwGwWLMKdIT24xhDf+mRoCehA8yUlLmuJgS9JeMt4IM=", + "lastModified": 1688484237, + "narHash": "sha256-qFUn2taHGe203wm7Oio4UGFz1sAiq+kitRexY3sQ1CA=", "owner": "nix-community", "repo": "fenix", - "rev": "37b3a6dad6d6060bd305eb7d3628d3b476c87bb6", + "rev": "626a9e0a84010728b335f14d3982e11b99af7dc6", "type": "github" }, "original": { @@ -78,11 +78,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1681202837, - "narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=", + "lastModified": 1689068808, + "narHash": "sha256-6ixXo3wt24N/melDWjq70UuHQLxGV8jZvooRanIHXw0=", "owner": "numtide", "repo": "flake-utils", - "rev": "cfacdce06f30d2b68473a46042957675eebb3401", + "rev": "919d646de7be200f3bf08cb76ae1f09402b6f9b4", "type": "github" }, "original": { @@ -93,11 +93,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1682600000, - "narHash": "sha256-ha4BehR1dh8EnXSoE1m/wyyYVvHI9txjW4w5/oxsW5Y=", + "lastModified": 1672580127, + "narHash": "sha256-3lW3xZslREhJogoOkjeZtlBtvFMyxHku7I/9IVehhT8=", "owner": "nixos", "repo": "nixpkgs", - "rev": "50fc86b75d2744e1ab3837ef74b53f103a9b55a0", + "rev": "0874168639713f547c05947c76124f78441ea46c", "type": "github" }, "original": { @@ -109,11 +109,11 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1683408522, - "narHash": "sha256-9kcPh6Uxo17a3kK3XCHhcWiV1Yu1kYj22RHiymUhMkU=", + "lastModified": 1688231357, + "narHash": "sha256-ZOn16X5jZ6X5ror58gOJAxPfFLAQhZJ6nOUeS4tfFwo=", "owner": "nixos", "repo": "nixpkgs", - "rev": "897876e4c484f1e8f92009fd11b7d988a121a4e7", + "rev": "645ff62e09d294a30de823cb568e9c6d68e92606", "type": "github" }, "original": { @@ -125,11 +125,11 @@ }, "nixpkgs_3": { "locked": { - "lastModified": 1683631309, - "narHash": "sha256-1cNXXM98+9NyH8TV3TYSESFjZ+MZGbFbNO4AtM6um3I=", + "lastModified": 1690294827, + "narHash": "sha256-JV53dEaMM566e+6R4Wj58jBAkFg7HaZr3SsXZ9hdh40=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "e840c93ea7623f31400bc8fbe1d4cc767becf34d", + "rev": "7ce0abe77d2ace6d6fc43ff7077019e62a77e741", "type": "github" }, "original": { @@ -150,11 +150,11 @@ "rust-analyzer-src": { "flake": false, "locked": { - "lastModified": 1683571499, - "narHash": "sha256-SUs1qlsGJB09yjAKLQIJVxUPHGUdcayzE9IOkV4XRFM=", + "lastModified": 1688410727, + "narHash": "sha256-TqKZO9D64UDBCMY2sUP2ebAKP0oY7S9enrHfZaDiqBQ=", "owner": "rust-lang", "repo": "rust-analyzer", - "rev": "c26a43d6bd660eba94500645a47f931e153015d8", + "rev": "45272efec5fcb8bc46e303d6ced8bd2ba095a667", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 3bf91b2..895303d 100644 --- a/flake.nix +++ b/flake.nix @@ -19,6 +19,7 @@ outputs = { self, nixpkgs, cargo2nix, flake-utils, fenix }: flake-utils.lib.eachSystem [ + "x86_64-linux" "x86_64-unknown-linux-musl" "aarch64-unknown-linux-musl" "armv6l-unknown-linux-musleabihf" @@ -51,10 +52,16 @@ ]; }; - shell = pkgs.mkShell { + pkgVanilla = import nixpkgs { system = "x86_64-linux"; }; + + shell = pkgVanilla.mkShell { buildInputs = [ - cargo2nix.packages.x86_64-linux.default + #cargo2nix.packages.x86_64-linux.default + fenix.packages.x86_64-linux.minimal.toolchain ]; + shellHook = '' + echo "AEROGRAME DEVELOPMENT SHELL ${fenix.packages.x86_64-linux.minimal.rustc}" + ''; }; rustTarget = if targetHost == "armv6l-unknown-linux-musleabihf" then "arm-unknown-linux-musleabihf" else targetHost; diff --git a/src/imap/mailbox_view.rs b/src/imap/mailbox_view.rs index 3625023..2124855 100644 --- a/src/imap/mailbox_view.rs +++ b/src/imap/mailbox_view.rs @@ -1,5 +1,3 @@ -use std::borrow::Cow; - use std::num::NonZeroU32; use std::sync::Arc; @@ -21,8 +19,7 @@ use imap_codec::types::sequence::{self, SequenceSet}; use eml_codec::{ imf::{self as imf}, part::{AnyPart}, - part::discrete::{Text, Binary}, - part::composite::{Message, Multipart}, + mime::r#type::Deductible, mime, }; @@ -619,6 +616,8 @@ fn string_to_flag(f: &str) -> Option { //@FIXME return an error if the envelope is invalid instead of panicking //@FIXME some fields must be defaulted if there are not set. fn message_envelope(msg: &imf::Imf) -> Envelope { + let from = msg.from.iter().map(convert_mbx).collect::>(); + Envelope { date: NString( msg.date.as_ref() @@ -628,9 +627,13 @@ fn message_envelope(msg: &imf::Imf) -> Envelope { msg.subject.as_ref() .map(|d| IString::try_from(d.to_string()).unwrap()), ), - from: msg.from.iter().map(convert_mbx).collect(), - sender: msg.sender.iter().map(convert_mbx).collect(), //@FIXME put from[0] if empty - reply_to: convert_addresses(&msg.reply_to), //@FIXME put from if empty + sender: msg.sender.as_ref().map(|v| vec![convert_mbx(v)]).unwrap_or(from.clone()), + reply_to: if msg.reply_to.is_empty() { + from.clone() + } else { + convert_addresses(&msg.reply_to) + }, + from: from, to: convert_addresses(&msg.to), cc: convert_addresses(&msg.cc), bcc: convert_addresses(&msg.bcc), @@ -681,10 +684,8 @@ b OK Fetch completed (0.001 + 0.000 secs). fn build_imap_email_struct<'a>(part: &AnyPart<'a>) -> Result { match part { AnyPart::Mult(x) => { - let subtype = x.interpreted.parsed.ctype.as_ref() - .map(|x| IString::try_from(String::from_utf8_lossy(x.sub).to_string()).ok()) - .flatten() - .unwrap_or(unchecked_istring("alternative")); + let itype = &x.mime.interpreted_type; + let subtype = IString::try_from(itype.subtype.to_string()).unwrap_or(unchecked_istring("alternative")); Ok(BodyStructure::Multi { bodies: x.children @@ -703,39 +704,39 @@ fn build_imap_email_struct<'a>(part: &AnyPart<'a>) -> Result { }) } AnyPart::Txt(x) => { - //@FIXME check if we must really guess a charset if none is provided, if so we must - //update this code - let basic = basic_fields(&x.interpreted.parsed)?; - - let subtype = x.interpreted.parsed.ctype.as_ref() - .map(|x| IString::try_from(String::from_utf8_lossy(x.sub).to_string()).ok()) - .flatten() - .unwrap_or(unchecked_istring("plain")); - - let number_of_lines = x.body.iter() - .filter(|x| **x == b'\n') - .count() - .try_into() - .unwrap_or(0); + let mut basic = basic_fields(&x.mime.fields, x.body.len())?; + // Get the interpreted content type, set it + let itype = match &x.mime.interpreted_type { + Deductible::Inferred(v) | Deductible::Explicit(v) => v + }; + let subtype = IString::try_from(itype.subtype.to_string()).unwrap_or(unchecked_istring("plain")); + + // Add charset to the list of parameters if we know it has been inferred as it will be + // missing from the parsed content. + if let Deductible::Inferred(charset) = &itype.charset { + basic.parameter_list.push(( + unchecked_istring("charset"), + IString::try_from(charset.to_string()).unwrap_or(unchecked_istring("us-ascii")) + )); + } Ok(BodyStructure::Single { body: FetchBody { basic, specific: SpecificFields::Text { subtype, - number_of_lines, + number_of_lines: nol(x.body), }, }, extension: None, }) } AnyPart::Bin(x) => { - //let (_, basic) = headers_to_basic_fields(part, bp.len())?; - let basic = basic_fields(&x.interpreted.parsed)?; - - let default = mime::r#type::NaiveType { main: &[], sub: &[], params: vec![] }; - let ct = x.interpreted.parsed.ctype.as_ref().unwrap_or(&default); + let basic = basic_fields(&x.mime.fields, x.body.len())?; + + let default = mime::r#type::NaiveType { main: &b"application"[..], sub: &b"octet-stream"[..], params: vec![] }; + let ct = x.mime.fields.ctype.as_ref().unwrap_or(&default); let type_ = IString::try_from(String::from_utf8_lossy(ct.main).to_string()) .or(Err(anyhow!("Unable to build IString from given Content-Type type given")))?; @@ -753,13 +754,7 @@ fn build_imap_email_struct<'a>(part: &AnyPart<'a>) -> Result { }) } AnyPart::Msg(x) => { - let basic = basic_fields(&x.interpreted.parsed)?; - - // We do not count the number of lines but the number of line - // feeds to have the same behavior as Dovecot and Cyrus. - // 2 lines = 1 line feed. - //let nol = inner.raw_message().iter().filter(|&c| c == &b'\n').count(); - let nol = 0; // @FIXME broken for now + let basic = basic_fields(&x.mime.fields, x.raw_part.len())?; Ok(BodyStructure::Single { body: FetchBody { @@ -767,7 +762,7 @@ fn build_imap_email_struct<'a>(part: &AnyPart<'a>) -> Result { specific: SpecificFields::Message { envelope: message_envelope(&x.imf), body_structure: Box::new(build_imap_email_struct(x.child.as_ref())?), - number_of_lines: u32::try_from(nol)?, + number_of_lines: nol(x.raw_part), }, }, extension: None, @@ -776,13 +771,21 @@ fn build_imap_email_struct<'a>(part: &AnyPart<'a>) -> Result { } } +fn nol(input: &[u8]) -> u32 { + input.iter() + .filter(|x| **x == b'\n') + .count() + .try_into() + .unwrap_or(0) +} + /// s is set to static to ensure that only compile time values /// checked by developpers are passed. fn unchecked_istring(s: &'static str) -> IString { IString::try_from(s).expect("this value is expected to be a valid imap-codec::IString") } -fn basic_fields(m: &mime::NaiveMIME) -> Result { +fn basic_fields(m: &mime::NaiveMIME, sz: usize) -> Result { let parameter_list = m.ctype .as_ref() .map(|x| x.params.iter() @@ -810,7 +813,7 @@ fn basic_fields(m: &mime::NaiveMIME) -> Result { _ => unchecked_istring("7bit"), }, // @FIXME we can't compute the size of the message currently... - size: u32::try_from(0)?, + size: u32::try_from(sz)?, }) } @@ -968,41 +971,42 @@ mod tests { #[test] fn fetch_body() -> Result<()> { let prefixes = [ + /* *** MY OWN DATASET *** */ "tests/emails/dxflrs/0001_simple", "tests/emails/dxflrs/0002_mime", "tests/emails/dxflrs/0003_mime-in-mime", "tests/emails/dxflrs/0004_msg-in-msg", - - // wrong. base64? + // eml_codec do not support continuation for the moment //"tests/emails/dxflrs/0005_mail-parser-readme", - "tests/emails/dxflrs/0006_single-mime", + "tests/emails/dxflrs/0007_raw_msg_in_rfc822", + + /* *** (STRANGE) RFC *** */ + //"tests/emails/rfc/000", // must return text/enriched, we return text/plain + //"tests/emails/rfc/001", // does not recognize the multipart/external-body, breaks the + // whole parsing + //"tests/emails/rfc/002", // wrong date in email + + //"tests/emails/rfc/003", // dovecot fixes \r\r: the bytes number is wrong + text/enriched + + /* *** THIRD PARTY *** */ + //"tests/emails/thirdparty/000", // dovecot fixes \r\r: the bytes number is wrong + //"tests/emails/thirdparty/001", // same + "tests/emails/thirdparty/002", // same + - // panic - thread 'imap::mailbox_view::tests::fetch_body' panicked at 'range end index 128 out of range for slice of length 127', src/imap/mailbox_view.rs:798:64 - //"tests/emails/dxflrs/0007_raw_msg_in_rfc822", - - // broken, wrong mimetype text, should be audio - // "tests/emails/rfc/000", - - // "tests/emails/rfc/001", // broken - // "tests/emails/rfc/002", // broken: dovecot adds \r when it is missing and count it as - // a character. Difference on how lines are counted too. - /*"tests/emails/rfc/003", // broken for the same reason - "tests/emails/thirdparty/000", - "tests/emails/thirdparty/001", - "tests/emails/thirdparty/002", - */ + /* *** LEGACY *** */ + //"tests/emails/legacy/000", // same issue with \r\r ]; for pref in prefixes.iter() { println!("{}", pref); let txt = fs::read(format!("{}.eml", pref))?; let exp = fs::read(format!("{}.dovecot.body", pref))?; - let message = eml_codec::email(&txt).unwrap(); + let message = eml_codec::parse_message(&txt).unwrap().1; let mut resp = Vec::new(); - MessageAttribute::Body(build_imap_email_struct(&message, message.root_part())?) - .encode(&mut resp); + MessageAttribute::Body(build_imap_email_struct(&message.child)?).encode(&mut resp).unwrap(); let resp_str = String::from_utf8_lossy(&resp).to_lowercase(); diff --git a/src/login/mod.rs b/src/login/mod.rs index 9df640b..a68cd9c 100644 --- a/src/login/mod.rs +++ b/src/login/mod.rs @@ -11,7 +11,7 @@ use k2v_client::{ }; use rand::prelude::*; use rusoto_core::HttpClient; -use rusoto_credential::{AwsCredentials, StaticProvider}; +use rusoto_credential::{StaticProvider}; use rusoto_s3::S3Client; use crate::cryptoblob::*; diff --git a/src/mail/mailbox.rs b/src/mail/mailbox.rs index 4a7d712..fe36a14 100644 --- a/src/mail/mailbox.rs +++ b/src/mail/mailbox.rs @@ -315,10 +315,9 @@ impl MailboxInternal { }, async { // Save mail meta - let mail_root = mail.parsed.imf; let meta = MailMeta { internaldate: now_msec(), - headers: vec![], + headers: mail.parsed.raw_headers.to_vec(), message_key: message_key.clone(), rfc822_size: mail.raw.len(), }; @@ -368,10 +367,9 @@ impl MailboxInternal { }, async { // Save mail meta - let mail_root = mail.parsed.imf; let meta = MailMeta { internaldate: now_msec(), - headers: vec![], //@FIXME we need to put the headers part + headers: mail.parsed.raw_headers.to_vec(), message_key: message_key.clone(), rfc822_size: mail.raw.len(), }; diff --git a/tests/parsing-crates/mail_parser.rs b/tests/parsing-crates/mail_parser.rs deleted file mode 100644 index df9b20b..0000000 --- a/tests/parsing-crates/mail_parser.rs +++ /dev/null @@ -1,73 +0,0 @@ -//use mail_parser_superboum::Message; // FAIL - -use mail_parser::Message; // PASS - -//use mail_parser_05::Message; // PASS -//use mail_parser_main::Message; // PASS -//use mail_parser_db61a03::Message; // PASS - -#[test] -fn test1() { - let input = br#"Content-Type: multipart/mixed; boundary="1234567890123456789012345678901234567890123456789012345678901234567890123456789012" - ---1234567890123456789012345678901234567890123456789012345678901234567890123456789012 -Content-Type: multipart/mixed; boundary="123456789012345678901234567890123456789012345678901234567890123456789012345678901" - ---123456789012345678901234567890123456789012345678901234567890123456789012345678901 -Content-Type: multipart/mixed; boundary="12345678901234567890123456789012345678901234567890123456789012345678901234567890" - ---12345678901234567890123456789012345678901234567890123456789012345678901234567890 -Content-Type: text/plain - -1 ---1234567890123456789012345678901234567890123456789012345678901234567890123456789012 -Content-Type: text/plain - -22 ---123456789012345678901234567890123456789012345678901234567890123456789012345678901 -Content-Type: text/plain - -333 ---12345678901234567890123456789012345678901234567890123456789012345678901234567890 -Content-Type: text/plain - -4444 -"#; - - let message = Message::parse(input); - dbg!(message); -} - -#[test] -fn test2() { - let input = br#"Content-Type: message/rfc822 - -Content-Type: message/rfc822 - -Content-Type: text/plain - -1"#; - - let message = Message::parse(input); - dbg!(message); -} - -#[test] -fn test3() { - let input = br#"Content-Type: multipart/mixed; boundary=":foo" - ---:foo ---:foo -Content-Type: text/plain ---:foo -Content-Type: text/plain ---:foo -Content-Type: text/html ---:foo-- - - -"#; - - let message = Message::parse(input); - dbg!(message); -} diff --git a/tests/parsing-crates/main.rs b/tests/parsing-crates/main.rs deleted file mode 100644 index 7ebf793..0000000 --- a/tests/parsing-crates/main.rs +++ /dev/null @@ -1 +0,0 @@ -mod mail_parser; -- cgit v1.2.3