Merge branch 'master' of git.deuxfleurs.fr:Deuxfleurs/infrastructure

24 files changed, 522 insertions, 0 deletions

diff --git a/app/seafile/build/mariadb/60-disable-dialog.cnf b/app/seafile/build/mariadb/60-disable-dialog.cnf
new file mode 100644
index 0000000..d41731a
--- /dev/null
+++ b/app/seafile/build/mariadb/60-disable-dialog.cnf
@@ -0,0 +1,3 @@
+[mariadb]
+pam_use_cleartext_plugin
+bind-address = 0.0.0.0
diff --git a/app/seafile/build/mariadb/60-ldap.cnf b/app/seafile/build/mariadb/60-ldap.cnf
new file mode 100644
index 0000000..72ffb9f
--- /dev/null
+++ b/app/seafile/build/mariadb/60-ldap.cnf
@@ -0,0 +1,3 @@
+[mariadb]
+plugin-load=auth_pam.so
+
diff --git a/app/seafile/build/mariadb/60-remote.cnf b/app/seafile/build/mariadb/60-remote.cnf
new file mode 100644
index 0000000..acf8f9b
--- /dev/null
+++ b/app/seafile/build/mariadb/60-remote.cnf
@@ -0,0 +1,2 @@
+[mysqld]
+bind-address = *
diff --git a/app/seafile/build/mariadb/Dockerfile b/app/seafile/build/mariadb/Dockerfile
new file mode 100644
index 0000000..15ef954
--- /dev/null
+++ b/app/seafile/build/mariadb/Dockerfile
@@ -0,0 +1,14 @@
+FROM debian:stretch
+
+RUN apt-get update && \
+    apt-get dist-upgrade -y && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y mariadb-server mariadb-client libnss-ldapd
+
+COPY 60-ldap.cnf /etc/mysql/mariadb.conf.d/60-ldap.cnf
+COPY 60-remote.cnf /etc/mysql/mariadb.conf.d/60-remote.cnf
+COPY 60-disable-dialog.cnf /etc/mysql/mariadb.conf.d/60-disable-dialog.cnf
+COPY pam-mariadb /etc/pam.d/mariadb 
+COPY nsswitch.conf /etc/nsswitch.conf
+COPY entrypoint.sh /usr/local/bin/entrypoint
+
+ENTRYPOINT ["/usr/local/bin/entrypoint"]
diff --git a/app/seafile/build/mariadb/README.md b/app/seafile/build/mariadb/README.md
new file mode 100644
index 0000000..1a3b8aa
--- /dev/null
+++ b/app/seafile/build/mariadb/README.md
@@ -0,0 +1,19 @@
+```
+sudo docker build -t superboum/amd64_mariadb:v3 .
+
+sudo docker run \
+  -t -i \
+  -p 3306:3306 \
+  -v /tmp/mysql:/var/lib/mysql \
+  -e LDAP_URI='ldap://bottin.service.2.cluster.deuxfleurs.fr' \
+  -e LDAP_BASE='ou=users,dc=deuxfleurs,dc=fr' \
+  -e LDAP_VERSION=3 \
+  -e LDAP_BIND_DN='cn=admin,dc=deuxfleurs,dc=fr' \
+  -e LDAP_BIND_PW='xxxx' \
+  -e MYSQL_PASSWORD='xxxx' \
+  superboum/amd64_mariadb:v1 \
+  tail -f /var/log/mysql/error.log
+
+CREATE USER quentin@localhost IDENTIFIED VIA pam USING 'mariadb';
+
+```
diff --git a/app/seafile/build/mariadb/entrypoint.sh b/app/seafile/build/mariadb/entrypoint.sh
new file mode 100755
index 0000000..7ebf049
--- /dev/null
+++ b/app/seafile/build/mariadb/entrypoint.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+
+set -e
+
+cat > /etc/nslcd.conf <<EOF
+# /etc/nslcd.conf
+# nslcd configuration file. See nslcd.conf(5)
+# for details.
+
+# The user and group nslcd should run as.
+uid nslcd
+gid nslcd
+
+# The location at which the LDAP server(s) should be reachable.
+uri ${LDAP_URI}
+
+# The search base that will be used for all queries.
+base ${LDAP_BASE}
+
+# The LDAP protocol version to use.
+ldap_version ${LDAP_VERSION}
+
+# The DN to bind with for normal lookups.
+binddn ${LDAP_BIND_DN}
+bindpw ${LDAP_BIND_PW}
+
+# The DN used for password modifications by root.
+#rootpwmoddn cn=admin,dc=example,dc=com
+
+# SSL options
+#ssl off
+#tls_reqcert never
+tls_cacertfile /etc/ssl/certs/ca-certificates.crt
+
+# The search scope.
+#scope sub
+EOF
+
+/usr/sbin/nslcd
+
+chown mysql:mysql /var/lib/mysql
+[ -z "$(ls -A /var/lib/mysql)" ] && mysql_install_db --user=mysql --basedir=/usr --datadir=/var/lib/mysql
+
+/usr/bin/mysqld_safe &
+
+until ls /var/run/mysqld/mysqld.sock; do sleep 1; done
+/usr/bin/mysqladmin -u root password ${MYSQL_PASSWORD} || true
+
+exec "$@"
+
diff --git a/app/seafile/build/mariadb/nsswitch.conf b/app/seafile/build/mariadb/nsswitch.conf
new file mode 100644
index 0000000..853348e
--- /dev/null
+++ b/app/seafile/build/mariadb/nsswitch.conf
@@ -0,0 +1,21 @@
+# /etc/nsswitch.conf
+#
+# Example configuration of GNU Name Service Switch functionality.
+# If you have the `glibc-doc-reference' and `info' packages installed, try:
+# `info libc "Name Service Switch"' for information about this file.
+
+passwd:         files ldap
+group:          files ldap
+shadow:         files ldap
+gshadow:        files
+
+hosts:          files dns
+networks:       files
+
+protocols:      db files
+services:       db files
+ethers:         db files
+rpc:            db files
+
+netgroup:       nis
+
diff --git a/app/seafile/build/mariadb/pam-mariadb b/app/seafile/build/mariadb/pam-mariadb
new file mode 100644
index 0000000..e1bb814
--- /dev/null
+++ b/app/seafile/build/mariadb/pam-mariadb
@@ -0,0 +1,2 @@
+auth            required        pam_ldap.so
+account         required        pam_ldap.so
diff --git a/app/seafile/build/seafile/Dockerfile b/app/seafile/build/seafile/Dockerfile
new file mode 100644
index 0000000..88dee4f
--- /dev/null
+++ b/app/seafile/build/seafile/Dockerfile
@@ -0,0 +1,46 @@
+FROM amd64/debian:buster as builder
+
+ENV VERSION 7.0.5
+
+RUN apt-get update && \
+    apt-get dist-upgrade -y && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y wget tar && \
+    wget https://download.seadrive.org/seafile-server_${VERSION}_x86-64.tar.gz -O ./seafile.tar.gz && \
+    tar xf ./seafile.tar.gz && \
+    mv seafile-server-${VERSION} seafile-server
+
+FROM amd64/debian:buster
+
+COPY --from=builder ./seafile-server /srv/webstore/seafile-server
+
+RUN apt-get update && \
+    apt-get dist-upgrade -y && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y \
+      python \
+      mariadb-client \
+      python2.7 \
+      libpython2.7 \
+      python-setuptools \
+      python-ldap \
+      python-urllib3 \
+      ffmpeg \
+      python-pip \
+      python-mysqldb \
+      python-memcache \
+      procps \
+      python-requests && \
+    pip install Pillow==4.3.0 && \
+    pip install moviepy && \
+    useradd -u 1000 -d /srv/webstore seauser && \
+    chown -R seauser:1000 /srv/webstore/
+
+RUN mkdir -p /usr/local/lib/mariadb/plugin/  && \
+    ln -s /usr/lib/x86_64-linux-gnu/mariadb*/plugin/mysql_clear_password.so /usr/local/lib/mariadb/plugin/ && \
+    ln -s /usr/lib/x86_64-linux-gnu/mariadb*/plugin/dialog.so /usr/local/lib/mariadb/plugin/
+    
+WORKDIR /srv/webstore/seafile-server
+COPY seadocker /usr/local/bin/seadocker
+COPY seaenv /usr/local/bin/seaenv
+
+ENTRYPOINT ["/usr/local/bin/seaenv"]
+CMD ["/usr/local/bin/seadocker"]
diff --git a/app/seafile/build/seafile/README.md b/app/seafile/build/seafile/README.md
new file mode 100644
index 0000000..26d04e0
--- /dev/null
+++ b/app/seafile/build/seafile/README.md
@@ -0,0 +1,27 @@
+
+```bash
+sudo docker build -t superboum/amd64_seafile:v5 .
+```
+
+When upgrading, connect on a production server and run:
+
+```bash
+nomad stop seafile
+sudo docker build -t superboum/amd64_seafile:v6 .
+
+sudo docker run -t -i \
+  -v /mnt/glusterfs/seafile:/mnt/seafile-data \
+  -v /mnt/glusterfs/seaconf/conf:/srv/webstore/conf \
+  -v /mnt/glusterfs/seaconf/ccnet:/srv/webstore/ccnet \
+  superboum/amd64_seafile:v5
+
+# See:
+#  * https://download.seafile.com/published/seafile-manual/deploy/upgrade.md
+#  * https://download.seafile.com/published/seafile-manual/changelog/server-changelog.md
+
+
+
+nomad start seafile.hcl
+```
+
+when upgrading, change the command on start
diff --git a/app/seafile/build/seafile/seadocker b/app/seafile/build/seafile/seadocker
new file mode 100755
index 0000000..5b5982b
--- /dev/null
+++ b/app/seafile/build/seafile/seadocker
@@ -0,0 +1,4 @@
+#!/bin/bash
+/srv/webstore/seafile-server/seafile.sh start
+/srv/webstore/seafile-server/seahub.sh start
+tail -f /srv/webstore/logs/*
diff --git a/app/seafile/build/seafile/seaenv b/app/seafile/build/seafile/seaenv
new file mode 100755
index 0000000..3b0e0bb
--- /dev/null
+++ b/app/seafile/build/seafile/seaenv
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+chown seauser /srv/webstore
+chown seauser -R /srv/webstore/ccnet
+chown seauser -R /srv/webstore/conf
+
+runuser -u seauser -- "$@"
diff --git a/app/seafile/config/ccnet/seafile.ini b/app/seafile/config/ccnet/seafile.ini
new file mode 100644
index 0000000..306d126
--- /dev/null
+++ b/app/seafile/config/ccnet/seafile.ini
@@ -0,0 +1 @@
+/mnt/seafile-data/
\ No newline at end of file
diff --git a/app/seafile/config/conf/ccnet.conf.tpl b/app/seafile/config/conf/ccnet.conf.tpl
new file mode 100644
index 0000000..2395a9b
--- /dev/null
+++ b/app/seafile/config/conf/ccnet.conf.tpl
@@ -0,0 +1,29 @@
+[General]
+USER_NAME = deuxfleurs
+ID = {{ key "secrets/seafile/ccnet/seafile_id" | trimSpace }}
+NAME = deuxfleurs
+SERVICE_URL = https://cloud.deuxfleurs.fr
+
+[Network]
+PORT = 10001
+
+[Client]
+PORT = 13418
+
+[LDAP]
+HOST = ldap://bottin2.service.2.cluster.deuxfleurs.fr/
+BASE = ou=users,dc=deuxfleurs,dc=fr
+USER_DN = {{ key "secrets/seafile/ccnet/ldap_binddn" | trimSpace }}
+FILTER = memberOf=CN=seafile,OU=groups,DC=deuxfleurs,DC=fr
+PASSWORD = {{ key "secrets/seafile/ccnet/ldap_bindpwd" | trimSpace }}
+LOGIN_ATTR = mail
+
+[Database]
+ENGINE = mysql
+HOST = mariadb.service.2.cluster.deuxfleurs.fr
+PORT = 3306
+USER = seafile
+PASSWD = {{ key "secrets/seafile/ccnet/mysql_pwd" | trimSpace }}
+DB = ccnet-db
+CONNECTION_CHARSET = utf8
+
diff --git a/app/seafile/config/conf/gunicorn.conf b/app/seafile/config/conf/gunicorn.conf
new file mode 100644
index 0000000..415fd32
--- /dev/null
+++ b/app/seafile/config/conf/gunicorn.conf
@@ -0,0 +1,16 @@
+import os
+
+daemon = True
+workers = 5
+
+# default localhost:8000
+bind = "[::]:8000"
+
+# Pid
+pids_dir = '/srv/webstore/pids'
+pidfile = os.path.join(pids_dir, 'seahub.pid')
+
+# for file upload, we need a longer timeout value (default is only 30s, too short)
+timeout = 1200
+
+limit_request_line = 8190
diff --git a/app/seafile/config/conf/seafdav.conf b/app/seafile/config/conf/seafdav.conf
new file mode 100644
index 0000000..af78547
--- /dev/null
+++ b/app/seafile/config/conf/seafdav.conf
@@ -0,0 +1,6 @@
+[WEBDAV]
+host = ::
+enabled = true
+port = 8084
+fastcgi = false
+share_name = /seafdav
diff --git a/app/seafile/config/conf/seafile.conf.tpl b/app/seafile/config/conf/seafile.conf.tpl
new file mode 100644
index 0000000..a6425e9
--- /dev/null
+++ b/app/seafile/config/conf/seafile.conf.tpl
@@ -0,0 +1,19 @@
+[network]
+port = 12001
+
+[fileserver]
+port = 8083
+max_upload_size=8192
+max_download_dir_size=8192
+
+[database]
+type = mysql
+host = mariadb.service.2.cluster.deuxfleurs.fr
+port = 3306
+user = seafile
+password = {{ key "secrets/seafile/ccnet/mysql_pwd" | trimSpace }}
+db_name = seafile-db
+connection_charset = utf8
+
+[quota]
+default = 50
diff --git a/app/seafile/config/conf/seahub_settings.py.tpl b/app/seafile/config/conf/seahub_settings.py.tpl
new file mode 100644
index 0000000..6c63ee4
--- /dev/null
+++ b/app/seafile/config/conf/seahub_settings.py.tpl
@@ -0,0 +1,21 @@
+SECRET_KEY = "8ep+sgi&s1-f2cq2178!ekk!0h0nw2y4z1-olbaopxmodsd8vk"
+FILE_SERVER_ROOT = 'https://cloud.deuxfleurs.fr/seafhttp'
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.mysql',
+        'NAME': 'seahub-db',
+        'USER': 'seafile',
+        'PASSWORD': '{{ key "secrets/seafile/ccnet/mysql_pwd" | trimSpace }}',
+        'HOST': 'mariadb.service.2.cluster.deuxfleurs.fr',
+        'PORT': '3306',
+        'OPTIONS': {
+            'init_command': 'SET storage_engine=INNODB',
+        }
+    }
+}
+FILE_PREVIEW_MAX_SIZE = 100 * 1024 * 1024
+ENABLE_THUMBNAIL = True
+THUMBNAIL_ROOT = '/mnt/seafile-data/thumbnail/thumb/'
+THUMBNAIL_EXTENSION = 'png'
+THUMBNAIL_DEFAULT_SIZE = '24'
+PREVIEW_DEFAULT_SIZE = '300'
diff --git a/app/seafile/config/mariadb/main/env.tpl b/app/seafile/config/mariadb/main/env.tpl
new file mode 100644
index 0000000..0fe903b
--- /dev/null
+++ b/app/seafile/config/mariadb/main/env.tpl
@@ -0,0 +1,6 @@
+LDAP_URI = "ldap://bottin2.service.2.cluster.deuxfleurs.fr"
+LDAP_BASE = "ou=users,dc=deuxfleurs,dc=fr"
+LDAP_VERSION = 3
+LDAP_BIND_DN = "{{ key "secrets/mariadb/main/ldap_binddn" | trimSpace }}"
+LDAP_BIND_PW = "{{ key "secrets/mariadb/main/ldap_bindpwd" | trimSpace }}"
+MYSQL_PASSWORD = "{{ key "secrets/mariadb/main/mysql_pwd" | trimSpace }}"
diff --git a/app/seafile/deploy/seafile.hcl b/app/seafile/deploy/seafile.hcl
new file mode 100644
index 0000000..d8488d2
--- /dev/null
+++ b/app/seafile/deploy/seafile.hcl
@@ -0,0 +1,222 @@
+job "seafile" {
+  datacenters = ["dc1"]
+  type = "service"
+  priority = 10
+
+  constraint {
+    attribute = "${attr.cpu.arch}"
+    value     = "amd64"
+  }
+
+  group "main" {
+    count = 1
+
+    network {
+      port "seafile-frontend_port" { static = 8000 }
+      port "seafile-seafhttp_port" { static = 8083 }
+      port "seafile-dav_port" { static = 8084 }
+      port "seafile-hack_port" { static = 8085 } 
+      port "mariadb_port" { static = 3306 }
+    }
+
+    task "mariadb" {
+      driver = "docker"
+      config {
+        image = "superboum/amd64_mariadb:v4"
+        network_mode = "host"
+        command = "tail"
+        ports = [ "mariadb_port" ]
+        args = [
+          "-f", "/var/log/mysql/error.log",
+        ]
+        volumes = [
+          "/mnt/glusterfs/mariadb/main/server:/var/lib/mysql",
+        ]
+      }
+
+      template {
+        data = file("../config/mariadb/main/env.tpl")
+        destination = "secrets/env"
+        env = true
+      }
+
+      resources {
+        memory = 800
+      }
+
+      service {
+        tags = ["mariadb"]
+        port = "mariadb_port"
+        address_mode = "host"
+        name = "mariadb"
+        check {
+          type = "tcp"
+          port = "mariadb_port"
+          interval = "60s"
+          timeout = "5s"
+          check_restart {
+            limit = 3
+            grace = "90s"
+            ignore_warnings = false
+          }
+        }
+      }
+    }
+
+
+    task "hack" {
+      driver = "docker"
+      config {
+        image = "alpine/socat:1.0.5"
+        network_mode = "host"
+        ports = [ "seafile-hack_port" ]
+        command = "tcp6-listen:8085,fork,reuseaddr"
+        args = [ "tcp-connect:127.0.0.1:8083" ]
+      }
+      resources {
+        memory = 10
+      }
+      service {
+        tags = [
+          "seafile",
+          "traefik.enable=true",
+          "traefik.frontend.entryPoints=https,http",
+          "traefik.frontend.rule=Host:cloud.deuxfleurs.fr;PathPrefixStrip:/seafhttp"
+
+        ]
+        port = "seafile-hack_port"
+        address_mode = "host"
+        name = "seafhttp"
+        check {
+          type = "tcp"
+          port = "seafile-hack_port"
+          interval = "60s"
+          timeout = "5s"
+          check_restart {
+            limit = 3
+            grace = "90s"
+            ignore_warnings = false
+          }
+        }
+      }
+
+    }
+       
+    task "server" {
+      driver = "docker"
+      config {
+        image = "superboum/amd64_seafile:v6"
+        network_mode = "host"
+        ports = [ "seafile-frontend_port", "seafile-dav_port", "seafile-seafhttp_port" ]
+        
+        ## cmd + args are used for running an instance attachable for update
+        # command = "/bin/sleep"
+        # args = ["999999"]
+
+        mounts = [
+          {
+            type = "bind"
+            source = "/mnt/glusterfs/seafile"
+            target = "/mnt/seafile-data"
+          }
+        ]
+
+        volumes = [
+          "secrets/conf:/srv/webstore/conf",
+          "secrets/ccnet:/srv/webstore/ccnet"
+        ]
+      }
+
+      resources {
+        memory = 600
+      }
+
+      service {
+        tags = [
+          "seafile",
+          "traefik.enable=true",
+          "traefik.frontend.entryPoints=https,http",
+          "traefik.frontend.rule=Host:cloud.deuxfleurs.fr;PathPrefix:/"
+        ]
+        port = "seafile-frontend_port"
+        address_mode = "host"
+        name = "seahub"
+        check {
+          type = "tcp"
+          port = "seafile-frontend_port"
+          interval = "60s"
+          timeout = "5s"
+          check_restart {
+            limit = 3
+            grace = "90s"
+            ignore_warnings = false
+          }
+        }
+      }
+
+      service {
+        tags = [
+          "seafile",
+          "traefik.enable=true",
+          "traefik.frontend.entryPoints=https,http",
+          "traefik.frontend.rule=Host:cloud.deuxfleurs.fr;PathPrefix:/seafdav"
+
+        ]
+        port = "seafile-dav_port"
+        address_mode = "host"
+        name = "seafdav"
+        check {
+          type = "tcp"
+          port = "seafile-dav_port"
+          interval = "60s"
+          timeout = "5s"
+          check_restart {
+            limit = 3
+            grace = "90s"
+            ignore_warnings = false
+          }
+        }
+      }
+
+      template {
+        data = file("../config/conf/ccnet.conf.tpl")
+        destination = "secrets/conf/ccnet.conf"
+      }
+
+      template {
+        data = file("../config/conf/seafile.conf.tpl")
+        destination = "secrets/conf/seafile.conf"
+      }
+
+      template {
+        data = file("../config/conf/seahub_settings.py.tpl")
+        destination = "secrets/conf/seahub_settings.py"
+      }
+
+      template {
+        data = file("../config/ccnet/seafile.ini")
+        destination = "secrets/ccnet/seafile.ini"
+      }
+      template {
+        data = file("../config/conf/seafdav.conf")
+        destination = "secrets/conf/seafdav.conf"
+      }
+      template {
+        data = file("../config/conf/gunicorn.conf")
+        destination = "secrets/conf/gunicorn.conf"
+      }
+
+      # ---- secrets ----
+      template {
+        data = "{{ key \"secrets/seafile/conf/mykey.peer\" }}"
+        destination = "secrets/ccnet/mykey.peer"
+      }
+
+      template {
+        data = "{{ key \"secrets/seafile/conf/mykey.peer\" }}"
+        destination = "secrets/conf/mykey.peer"
+      }
+    }
+  }
+}
+
diff --git a/app/seafile/secrets/mariadb/main/ldap_binddn b/app/seafile/secrets/mariadb/main/ldap_binddn
new file mode 100644
index 0000000..e77ff39
--- /dev/null
+++ b/app/seafile/secrets/mariadb/main/ldap_binddn
@@ -0,0 +1 @@
+SERVICE_DN mysql MySQL/MariaDB database
diff --git a/app/seafile/secrets/mariadb/main/ldap_bindpwd b/app/seafile/secrets/mariadb/main/ldap_bindpwd
new file mode 100644
index 0000000..c29f983
--- /dev/null
+++ b/app/seafile/secrets/mariadb/main/ldap_bindpwd
@@ -0,0 +1 @@
+SERVICE_PASSWORD mysql
diff --git a/app/seafile/secrets/mariadb/main/mysql_pwd b/app/seafile/secrets/mariadb/main/mysql_pwd
new file mode 100644
index 0000000..ae7fd75
--- /dev/null
+++ b/app/seafile/secrets/mariadb/main/mysql_pwd
@@ -0,0 +1 @@
+USER mysql_pwd (what is this?)
diff --git a/app/seafile/secrets/seafile/conf/mykey.peer b/app/seafile/secrets/seafile/conf/mykey.peer
new file mode 100644
index 0000000..12f0e5f
--- /dev/null
+++ b/app/seafile/secrets/seafile/conf/mykey.peer
@@ -0,0 +1 @@
+USER Seafile peer key