Merge pull request 'Implement imap-flow' (#34) from refactor/imap-flow into main

Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/aerogramme/pulls/34

10 files changed, 276 insertions, 0 deletions

diff --git a/tests/instrumentation/README.md b/tests/instrumentation/README.md
new file mode 100644
index 0000000..b479cf4
--- /dev/null
+++ b/tests/instrumentation/README.md
@@ -0,0 +1,38 @@
+# Spawn Dovecot+Maddy+Cyrus
+
+Run:
+
+```
+docker-compose up
+```
+
+  - Dovecot
+    - listen on :993, run `openssl s_client -connect 127.0.0.1:993`
+    - login with `A LOGIN test pass`
+  - Maddy
+    - listen on :994,  run `openssl s_client -connect 127.0.0.1:994`
+    - login with `A LOGIN test@example.com pass`
+  - Cyrus
+    - listen on :143, run `nc 127.0.0.1 143`
+    - login with `A LOGIN test pass`
+  - Stalwart
+    - listen on :1993, run `openssl s_client -connect 127.0.0.1:993`
+    - login with `A LOGIN test@example.com pass`
+    - note: not packaged in docker yet...
+
+Other IMAP servers we could add:
+  - WildDuck (own node.js imap implementation)
+    - https://github.com/nodemailer/wildduck
+  - DBMail (own C IMAP implementation)
+    - https://github.com/dbmail/dbmail/tree/master
+  - UW IMAP (known to be the reference IMAP implementation)
+    - https://wiki.archlinux.org/title/UW_IMAP
+  - Apache James (has its own implementation of IMAP too)
+    - https://james.apache.org/
+  - Citadel
+    - https://citadel.org
+    - https://code.citadel.org/?p=citadel;a=tree;f=citadel/server/modules/imap;h=3ceaa1d6b518bddb7539911a8dd9d81136d4e594;hb=HEAD
+
+# Inject emails and dump the computed `BODY` + `BODYSTRUCTURE`
+
+Once you ran `docker-compose up`, launch `./send-to-imap.py`
diff --git a/tests/instrumentation/docker-compose.yml b/tests/instrumentation/docker-compose.yml
new file mode 100644
index 0000000..6a112bb
--- /dev/null
+++ b/tests/instrumentation/docker-compose.yml
@@ -0,0 +1,22 @@
+version: '3.4'
+services:
+  dovecot:
+    image: dovecot/dovecot:2.3.19.1
+    ports:
+      - "993:993/tcp"
+
+  maddy:
+    build:
+      context: ./docker/maddy/
+    image: maddy
+    ports:
+      - "994:993/tcp"
+
+  cyrus:
+    build:
+      context: ./docker/cyrus/
+    image: cyrus
+    volumes:
+      - "/dev/log:/dev/log"
+    ports:
+      - "143:143/tcp"
diff --git a/tests/instrumentation/docker/cyrus/Dockerfile b/tests/instrumentation/docker/cyrus/Dockerfile
new file mode 100644
index 0000000..67b4e11
--- /dev/null
+++ b/tests/instrumentation/docker/cyrus/Dockerfile
@@ -0,0 +1,15 @@
+FROM debian:buster
+
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt update && \
+    apt install -y cyrus-imapd cyrus-pop3d cyrus-nntpd cyrus-caldav cyrus-admin sasl2-bin && \
+    echo "admins: cyrus" >> /etc/imapd.conf && \
+    touch /var/lib/cyrus/tls_sessions.db && \
+    chown cyrus:mail /var/lib/cyrus/tls_sessions.db && \
+    mkdir /run/cyrus && \
+    chown -R cyrus:mail /run/cyrus
+
+COPY entrypoint.sh /usr/local/bin/entrypoint.sh
+
+CMD [ "/usr/local/bin/entrypoint.sh" ]
+
diff --git a/tests/instrumentation/docker/cyrus/entrypoint.sh b/tests/instrumentation/docker/cyrus/entrypoint.sh
new file mode 100755
index 0000000..c410e9d
--- /dev/null
+++ b/tests/instrumentation/docker/cyrus/entrypoint.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+cyrmaster -D -l 32 -C /etc/imapd.conf -M /etc/cyrus.conf &
+sleep 2
+
+echo cyrus | saslpasswd2 -p cyrus 
+echo pass | saslpasswd2 -p test
+
+cyradm -u cyrus -w cyrus 127.0.0.1 <<EOF
+cm user.test
+setaclmailbox user.test test kxtelrswip
+exit
+EOF
+
+wait
diff --git a/tests/instrumentation/docker/maddy/Dockerfile b/tests/instrumentation/docker/maddy/Dockerfile
new file mode 100644
index 0000000..341e398
--- /dev/null
+++ b/tests/instrumentation/docker/maddy/Dockerfile
@@ -0,0 +1,16 @@
+FROM debian:buster as builder
+
+WORKDIR /root
+RUN apt-get update && apt-get install -y openssl && \
+    openssl req -nodes -new -x509 -subj "/C=DW/ST=Sto/L=Ankh-Morpork /O=Unseen University/OU=Library/CN=Ook/emailAddress=ook@ook.ook" -keyout privkey.pem -out fullchain.pem
+
+FROM foxcpp/maddy:0.6.2
+
+COPY --from=builder /root/privkey.pem /data/tls/privkey.pem
+COPY --from=builder /root/fullchain.pem /data/tls/fullchain.pem
+
+ENV MADDY_DOMAIN=example.com MADDY_HOSTNAME=mx.example.com
+
+COPY entrypoint.sh /entrypoint.sh
+
+ENTRYPOINT [ "/entrypoint.sh" ]
diff --git a/tests/instrumentation/docker/maddy/entrypoint.sh b/tests/instrumentation/docker/maddy/entrypoint.sh
new file mode 100755
index 0000000..f875624
--- /dev/null
+++ b/tests/instrumentation/docker/maddy/entrypoint.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+maddy -config /data/maddy.conf run &
+
+sleep 2
+maddyctl creds create --password pass test@example.com
+maddyctl imap-acct create test@example.com
+
+wait
diff --git a/tests/instrumentation/inject_emails.sh b/tests/instrumentation/inject_emails.sh
new file mode 100755
index 0000000..a021170
--- /dev/null
+++ b/tests/instrumentation/inject_emails.sh
@@ -0,0 +1,21 @@
+#!/bin/sh
+
+SCRIPT=$0
+USER=$1
+SUBFOLDER=$2
+cd $(dirname $SCRIPT)
+
+function mail_lmtp_session (
+	echo -e "LHLO localhost\r"
+	for mail in $(find emails/$SUBFOLDER -name '*.eml' ); do
+		echo -e "MAIL FROM: <alex@adnab.me>\r"
+		echo -e "RCPT TO: $USER\r"
+		echo -e "DATA\r"
+		cat $mail
+		echo -e "\r"
+		echo -e ".\r"
+	done
+	echo -e "QUIT\r"
+)
+
+mail_lmtp_session | tee >(nc localhost 12024)
diff --git a/tests/instrumentation/rm-mail-parser-expected-struct.py b/tests/instrumentation/rm-mail-parser-expected-struct.py
new file mode 100644
index 0000000..3dbc56e
--- /dev/null
+++ b/tests/instrumentation/rm-mail-parser-expected-struct.py
@@ -0,0 +1,17 @@
+from os import listdir
+from os.path import isfile, join
+import sys
+
+path = sys.argv[1]
+onlyfiles = [join(path, f) for f in listdir(path) if isfile(join(path, f)) and len(f) > 4 and f[-4:] == ".txt"]
+
+for p in onlyfiles:
+    g = p[:-4] + ".eml"
+    print(f"{p} -> {g}")
+    with open(p, 'r+b') as inp:
+        with open(g, 'w+b') as out:
+            for line in inp:
+                if b"EXPECTED STRUCTURE" in line:
+                    break
+                out.write(line)
+            
diff --git a/tests/instrumentation/send-to-imap.py b/tests/instrumentation/send-to-imap.py
new file mode 100644
index 0000000..df49076
--- /dev/null
+++ b/tests/instrumentation/send-to-imap.py
@@ -0,0 +1,108 @@
+from imaplib import IMAP4_SSL, IMAP4
+from os import listdir
+from os.path import isfile, join
+import sys
+
+# COMMAND USAGE
+#
+# start a test IMAP servers:
+#   docker-compose.up
+# then call this script. eg:
+#   ./send-to-imap.py all ./emails/dxflrs/
+
+
+def rebuild_body_res(b):
+    bb = b''
+    for e in b:
+        if type(e) is tuple:
+            bb += b'\r\n'.join([p for p in e])
+        else:
+            bb += e
+
+    f = bb[bb.find(b'('):]
+    return f
+
+mode = sys.argv[1]
+path = sys.argv[2]
+
+base_test_mb = "kzUXL7HyS5OjLcU8"
+parameters = {
+  "dovecot": {
+    "con": IMAP4_SSL,
+    "port": 993,
+    "user": "test",
+    "pw": "pass",
+    "ext": ".dovecot",
+    "mb": base_test_mb,
+  },
+  "maddy": {
+    "con": IMAP4_SSL,
+    "port": 994,
+    "user": "test@example.com",
+    "pw": "pass",
+    "ext": ".maddy",
+    "mb": base_test_mb,
+  },
+  "cyrus": {
+    "con": IMAP4,
+    "port": 143,
+    "user": "test",
+    "pw": "pass",
+    "ext": ".cyrus",
+    "mb": "INBOX."+base_test_mb,
+  },
+  "stalwart": {
+    "con": IMAP4_SSL,
+    "port": 1993,
+    "user": "test@example.com",
+    "pw": "pass",
+    "ext": ".stalwart.0.2.0",
+    "mb": base_test_mb,
+  }
+}
+
+queue = list(parameters.keys())
+if mode in parameters:
+    queue = [ mode ]
+
+onlyfiles = [join(path, f) for f in listdir(path) if isfile(join(path, f)) and len(f) > 4 and f[-4:] == ".eml"]
+
+for target in queue:
+    print(f"--- {target} ---")
+    conf = parameters[target]
+    test_mb = conf['mb']
+
+    with conf['con'](host="localhost", port=conf['port']) as M:
+        print(M.login(conf['user'], conf['pw']))
+        print(M.delete(test_mb))
+        print(M.create(test_mb))
+
+
+        print(M.list())
+        print(M.select(test_mb))
+        failed = 0
+        for (idx, f) in enumerate(onlyfiles):
+            f_noext = f[:-4]
+            try:
+                with open(f, 'r+b') as mail:
+                    print(M.append(test_mb, [], None, mail.read()))
+                    seq = (f"{idx+1-failed}:{idx+1-failed}").encode()
+                    (r, b) = M.fetch(seq, "(BODY)")
+                    print((r, b))
+                    assert r == 'OK'
+            
+
+                    with open(f_noext + conf['ext'] + ".body", 'w+b') as w:
+                        w.write(rebuild_body_res(b))
+
+                    (r, b) = M.fetch(seq, "(BODYSTRUCTURE)")
+                    print((r, b))
+                    assert r == 'OK'
+                    with open(f_noext + conf['ext'] + ".bodystructure", 'w+b') as w:
+                        w.write(rebuild_body_res(b))
+            except:
+                failed += 1
+                print(f"failed {f}")
+
+        M.close()
+        M.logout()
diff --git a/tests/instrumentation/unix2dos.py b/tests/instrumentation/unix2dos.py
new file mode 100755
index 0000000..4e6c9cf
--- /dev/null
+++ b/tests/instrumentation/unix2dos.py
@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+import sys
+
+buf = ""
+with open(sys.argv[1], 'r+b') as f:
+    buf = f.read()
+
+if buf.find(b'\r\n'):
+    print(f"{sys.argv[1]} is already a CRLF file")
+    sys.exit(1)
+
+buf = buf.replace(b'\n', b'\r\n')
+
+with open(sys.argv[1], 'w+b') as f:
+    f.write(buf)