From 8d17a07c9be5cd9d400644c34ea50177535d15f6 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Sat, 24 Dec 2022 22:59:37 +0100 Subject: reorganize some things --- README.md | 38 +- cluster/prod/app/secretmgr | 1 + cluster/prod/app/secretmgr.py | 1 - cluster/prod/app/shell.nix | 1 - cluster/staging/app/secretmgr | 1 + cluster/staging/app/secretmgr.py | 1 - cluster/staging/app/shell.nix | 1 - doc/example-hardware-configuration.nix | 38 -- doc/nixos-install-luks.md | 182 ---------- experimental/app/csi-s3/deploy/csi-s3.hcl | 39 +++ experimental/app/csi-s3/deploy/dummy-volume.hcl | 20 ++ experimental/app/nextcloud/config/litestream.yml | 10 + experimental/app/nextcloud/deploy/nextcloud.hcl | 137 ++++++++ .../app/nextcloud/secrets/nextcloud/admin_pass | 1 + .../app/nextcloud/secrets/nextcloud/admin_user | 1 + .../app/nextcloud/secrets/nextcloud/s3_access_key | 1 + .../app/nextcloud/secrets/nextcloud/s3_secret_key | 1 + experimental/app/ssb/deploy/go-ssb-room.hcl | 83 +++++ experimental/app/ssb/deploy/ssb-room.hcl | 59 ++++ .../app/telemetry-elastic/config/apm-config.yaml | 20 ++ .../app/telemetry-elastic/config/filebeat.yml | 46 +++ .../config/grafana-litestream.yml | 10 + .../grafana/provisioning/datasources/elastic.yaml | 21 ++ .../app/telemetry-elastic/config/otel-config.yaml | 56 +++ .../telemetry-elastic/deploy/telemetry-system.hcl | 182 ++++++++++ .../app/telemetry-elastic/deploy/telemetry.hcl | 181 ++++++++++ experimental/app/yugabyte/deploy/yugabyte.hcl | 204 +++++++++++ experimental/bad.csi-s3/deploy/csi-s3.hcl | 39 --- experimental/bad.csi-s3/deploy/dummy-volume.hcl | 20 -- experimental/bad.nextcloud/config/litestream.yml | 10 - experimental/bad.nextcloud/deploy/nextcloud.hcl | 137 -------- .../bad.nextcloud/secrets/nextcloud/admin_pass | 1 - .../bad.nextcloud/secrets/nextcloud/admin_user | 1 - .../bad.nextcloud/secrets/nextcloud/s3_access_key | 1 - .../bad.nextcloud/secrets/nextcloud/s3_secret_key | 1 - experimental/bad.ssb/deploy/go-ssb-room.hcl | 83 ----- experimental/bad.ssb/deploy/ssb-room.hcl | 59 ---- .../bad.telemetry-elastic/config/apm-config.yaml | 20 -- .../bad.telemetry-elastic/config/filebeat.yml | 46 --- .../config/grafana-litestream.yml | 10 - .../grafana/provisioning/datasources/elastic.yaml | 21 -- .../bad.telemetry-elastic/config/otel-config.yaml | 56 --- .../deploy/telemetry-system.hcl | 182 ---------- .../bad.telemetry-elastic/deploy/telemetry.hcl | 181 ---------- experimental/bad.yugabyte/deploy/yugabyte.hcl | 204 ----------- .../luks-fde/example-hardware-configuration.nix | 38 ++ experimental/luks-fde/nixos-install-luks.md | 182 ++++++++++ secretmgr/secretmgr | 381 +++++++++++++++++++++ secretmgr/secretmgr.py | 380 -------------------- secretmgr/shell.nix | 15 - sshtool | 5 + 51 files changed, 1709 insertions(+), 1700 deletions(-) create mode 120000 cluster/prod/app/secretmgr delete mode 120000 cluster/prod/app/secretmgr.py delete mode 120000 cluster/prod/app/shell.nix create mode 120000 cluster/staging/app/secretmgr delete mode 120000 cluster/staging/app/secretmgr.py delete mode 120000 cluster/staging/app/shell.nix delete mode 100644 doc/example-hardware-configuration.nix delete mode 100644 doc/nixos-install-luks.md create mode 100644 experimental/app/csi-s3/deploy/csi-s3.hcl create mode 100644 experimental/app/csi-s3/deploy/dummy-volume.hcl create mode 100644 experimental/app/nextcloud/config/litestream.yml create mode 100644 experimental/app/nextcloud/deploy/nextcloud.hcl create mode 100644 experimental/app/nextcloud/secrets/nextcloud/admin_pass create mode 100644 experimental/app/nextcloud/secrets/nextcloud/admin_user create mode 100644 experimental/app/nextcloud/secrets/nextcloud/s3_access_key create mode 100644 experimental/app/nextcloud/secrets/nextcloud/s3_secret_key create mode 100644 experimental/app/ssb/deploy/go-ssb-room.hcl create mode 100644 experimental/app/ssb/deploy/ssb-room.hcl create mode 100644 experimental/app/telemetry-elastic/config/apm-config.yaml create mode 100644 experimental/app/telemetry-elastic/config/filebeat.yml create mode 100644 experimental/app/telemetry-elastic/config/grafana-litestream.yml create mode 100644 experimental/app/telemetry-elastic/config/grafana/provisioning/datasources/elastic.yaml create mode 100644 experimental/app/telemetry-elastic/config/otel-config.yaml create mode 100644 experimental/app/telemetry-elastic/deploy/telemetry-system.hcl create mode 100644 experimental/app/telemetry-elastic/deploy/telemetry.hcl create mode 100644 experimental/app/yugabyte/deploy/yugabyte.hcl delete mode 100644 experimental/bad.csi-s3/deploy/csi-s3.hcl delete mode 100644 experimental/bad.csi-s3/deploy/dummy-volume.hcl delete mode 100644 experimental/bad.nextcloud/config/litestream.yml delete mode 100644 experimental/bad.nextcloud/deploy/nextcloud.hcl delete mode 100644 experimental/bad.nextcloud/secrets/nextcloud/admin_pass delete mode 100644 experimental/bad.nextcloud/secrets/nextcloud/admin_user delete mode 100644 experimental/bad.nextcloud/secrets/nextcloud/s3_access_key delete mode 100644 experimental/bad.nextcloud/secrets/nextcloud/s3_secret_key delete mode 100644 experimental/bad.ssb/deploy/go-ssb-room.hcl delete mode 100644 experimental/bad.ssb/deploy/ssb-room.hcl delete mode 100644 experimental/bad.telemetry-elastic/config/apm-config.yaml delete mode 100644 experimental/bad.telemetry-elastic/config/filebeat.yml delete mode 100644 experimental/bad.telemetry-elastic/config/grafana-litestream.yml delete mode 100644 experimental/bad.telemetry-elastic/config/grafana/provisioning/datasources/elastic.yaml delete mode 100644 experimental/bad.telemetry-elastic/config/otel-config.yaml delete mode 100644 experimental/bad.telemetry-elastic/deploy/telemetry-system.hcl delete mode 100644 experimental/bad.telemetry-elastic/deploy/telemetry.hcl delete mode 100644 experimental/bad.yugabyte/deploy/yugabyte.hcl create mode 100644 experimental/luks-fde/example-hardware-configuration.nix create mode 100644 experimental/luks-fde/nixos-install-luks.md create mode 100755 secretmgr/secretmgr delete mode 100755 secretmgr/secretmgr.py delete mode 100644 secretmgr/shell.nix diff --git a/README.md b/README.md index c86a067..3527dbb 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,35 @@ This repository contains code to run Deuxfleur's infrastructure on NixOS. -It sets up the following: +## Our abstraction stack -- A Wireguard mesh between all nodes -- Consul, with TLS -- Nomad, with TLS +We try to build a generic abstraction stack between our different resources (CPU, RAM, disk, etc.) and our services (Chat, Storage, etc.), we develop our own tools when needed. + +Our first abstraction level is the NixOS level, which installs a bunch of standard components: + + * **Wireguard:** provides encrypted communication between remote nodes + * **Nomad:** schedule containers and handle their lifecycle + * **Consul:** distributed key value store + lock + service discovery + * **Docker:** package, distribute and isolate applications + +Then, inside our Nomad+Consul orchestrator, we deploy a number of base services: + + * **[Garage](https://git.deuxfleurs.fr/Deuxfleurs/garage/):** S3-compatible lightweight object store for self-hosted geo-distributed deployments (we also have a legacy glusterfs cluster) + * **[DiploNAT](https://git.deuxfleurs.fr/Deuxfleurs/diplonat):** network automation (firewalling, upnp igd) + * **[Bottin](https://git.deuxfleurs.fr/Deuxfleurs/bottin):** authentication and authorization (LDAP protocol, consul backend) + * **[Guichet](https://git.deuxfleurs.fr/Deuxfleurs/guichet):** a dashboard for our users and administrators + * **Stolon + PostgreSQL:** distributed relational database + * **Prometheus + Grafana:** monitoring + +Some services we provide based on this abstraction: + + * **Websites:** Garage (static) + fediverse blog (Plume) + * **Chat:** Synapse + Element Web (Matrix protocol) + * **Email:** Postfix SMTP + Dovecot IMAP + opendkim DKIM + Sogo webmail | Alps webmail (experimental) + * **Visioconference:** Jitsi + * **Collaboration:** CryptPad + +As a generic abstraction is provided, deploying new services should be easy. ## How to use this? @@ -16,11 +40,7 @@ See the following documentation topics: - [How to add new nodes to a cluster (rapid overview)](doc/adding-nodes.md) - [Architecture of this repo, how the scripts work](doc/architecture.md) - [List of TCP and UDP ports used by services](doc/ports) - -Additionnal documentation topics: - -- [Succint guide for NixOS installation with LUKX full disk encryption](doc/nixos-install-luks.md) (we don't do that in practice on our servers) -- [Example `hardware-config.nix` for a full disk encryption scenario](doc/example-hardware-configuration.nix) - [Why not Ansible?](doc/why-not-ansible.md) + diff --git a/cluster/prod/app/secretmgr b/cluster/prod/app/secretmgr new file mode 120000 index 0000000..6aff4ad --- /dev/null +++ b/cluster/prod/app/secretmgr @@ -0,0 +1 @@ +../../../secretmgr/secretmgr \ No newline at end of file diff --git a/cluster/prod/app/secretmgr.py b/cluster/prod/app/secretmgr.py deleted file mode 120000 index 107653c..0000000 --- a/cluster/prod/app/secretmgr.py +++ /dev/null @@ -1 +0,0 @@ -../../../secretmgr/secretmgr.py \ No newline at end of file diff --git a/cluster/prod/app/shell.nix b/cluster/prod/app/shell.nix deleted file mode 120000 index b10effc..0000000 --- a/cluster/prod/app/shell.nix +++ /dev/null @@ -1 +0,0 @@ -../../../secretmgr/shell.nix \ No newline at end of file diff --git a/cluster/staging/app/secretmgr b/cluster/staging/app/secretmgr new file mode 120000 index 0000000..6aff4ad --- /dev/null +++ b/cluster/staging/app/secretmgr @@ -0,0 +1 @@ +../../../secretmgr/secretmgr \ No newline at end of file diff --git a/cluster/staging/app/secretmgr.py b/cluster/staging/app/secretmgr.py deleted file mode 120000 index 107653c..0000000 --- a/cluster/staging/app/secretmgr.py +++ /dev/null @@ -1 +0,0 @@ -../../../secretmgr/secretmgr.py \ No newline at end of file diff --git a/cluster/staging/app/shell.nix b/cluster/staging/app/shell.nix deleted file mode 120000 index b10effc..0000000 --- a/cluster/staging/app/shell.nix +++ /dev/null @@ -1 +0,0 @@ -../../../secretmgr/shell.nix \ No newline at end of file diff --git a/doc/example-hardware-configuration.nix b/doc/example-hardware-configuration.nix deleted file mode 100644 index 0a72bd1..0000000 --- a/doc/example-hardware-configuration.nix +++ /dev/null @@ -1,38 +0,0 @@ -# Do not modify this file! It was generated by ‘nixos-generate-config’ -# and may be overwritten by future invocations. Please make changes -# to /etc/nixos/configuration.nix instead. -{ config, lib, pkgs, modulesPath, ... }: - -{ - imports = - [ (modulesPath + "/installer/scan/not-detected.nix") - ]; - - boot.initrd.availableKernelModules = [ "xhci_pci" "ehci_pci" "ahci" "usb_storage" "usbhid" "sd_mod" ]; - boot.initrd.kernelModules = [ "dm-snapshot" ]; - boot.kernelModules = [ "kvm-intel" ]; - boot.extraModulePackages = [ ]; - - boot.initrd.luks.devices."cryptlvm" = { - device = "/dev/disk/by-uuid/5d4fcef7-433d-43a9-be26-be940ce291c0"; - allowDiscards = true; - }; - - fileSystems."/" = - { device = "/dev/disk/by-uuid/5bad79e1-fdbf-48f3-861f-6810adc76195"; - fsType = "ext4"; - options = [ "relatime" "discard" ]; - }; - - fileSystems."/boot" = - { device = "/dev/disk/by-uuid/07E8-5958"; - fsType = "vfat"; - }; - - swapDevices = - [ { device = "/dev/disk/by-uuid/a3e0bae8-8b07-4e66-a4a7-6955639f2155"; } - ]; - - hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; -} - diff --git a/doc/nixos-install-luks.md b/doc/nixos-install-luks.md deleted file mode 100644 index 3f0feca..0000000 --- a/doc/nixos-install-luks.md +++ /dev/null @@ -1,182 +0,0 @@ -## Preparation - -Download NixOS 21.11 ISO. Burn to USB. - -## Booting into install environment - -Boot the ISO on PC to install. - -Become root with `sudo su` - -```bash -loadkeys fr -setfont sun12x22 -``` - -Do network config if necessary, see [install guide](https://nixos.org/manual/nixos/stable/index.html#sec-installation-booting-networking) - -## Make partitions - -```bash -cgdisk /dev/sda -``` - -Recommended layout: - -``` -/dev/sda1 512M ef00 EFI System partition -/dev/sda2 100% 8309 Linux LUKS -``` - -## Setup cryptography - -```bash -cryptsetup luksFormat /dev/sda2 -cryptsetup open /dev/sda2 cryptlvm -``` - -## Create PV, VG and LVs - -```bash -pvcreate /dev/mapper/cryptlvm -vgcreate NixosVG /dev/mapper/cryptlvm -lvcreate -L 8G NixosVG -n swap -lvcreate -l 100%FREE NixosVG -n root -``` - -## Format partitions - -```bash -mkfs.fat -F 32 -n boot /dev/sda1 -mkswap /dev/NixosVG/swap -mkfs.ext4 /dev/NixosVG/root -``` - -## Mount partitions - -```bash -swapon /dev/NixosVG/swap -mount /dev/NixosVG/root /mnt -mkdir /mnt/boot -mount /dev/sda1 /mnt/boot -``` - -## Generate base NixOS configuration - -```bash -nixos-generate-config --root /mnt -``` - -## Update `hardware-configuration.nix` - -This section is needed: - -```nix - boot.initrd.luks.devices."cryptlvm" = { - device = "/dev/disk/by-uuid/"; - allowDiscards = true; - }; -``` - -And for the root filesystem, remember to add the `relatime` and `discard` options so that it looks like this: - -```nix - fileSystems."/" = - { device = "/dev/disk/by-uuid/<...>"; - fsType = "ext4"; - options = [ "relatime" "discard" ]; - }; -``` - -## Update `configuration.nix` - -Just enough so that basic tasks can be done from keyboard and remotely: - -- timezone -- keyboard layout -- font `sun12x22` -- vim -- non-root user -- ssh -- tcp port 22 in firewall - -## Do the installation - -```bash -nixos-install -``` - -## First boot - -Reboot machine. Login as `root` - -```bash -passwd -``` - -If necessary, assign static IP. E.g. `ip addr add 192.168.1.40/24 dev eno1` or sth (replace ip and device appropriately) - -Remotely: `ssh-copy-id @`. Check SSH access is good. - -## Deploy from this repo - -See [this documentation](quick-start.md). - -## Old guide - -It's time! - -**Files in this repo to create/change:** - -- create node `.nix` file and symlink for node `.site.nix` (create site and - cluster `.nix` files if necessary; use existing files of e.g. the staging - cluster as examples/templates) -- make sure values are filled in correctly -- add node to `ssh_config` with it's LAN IP, we don't have VPN at this stage - -**Configuration steps on the node:** - -```bash -# On node being installed -mkdir -p /var/lib/deuxfleurs/remote-unlock -cd /var/lib/deuxfleurs/remote-unlock -ssh-keygen -t ed25519 -N "" -f ./ssh_host_ed25519_key -``` - -**Try to deploy:** - -```bash -# In nixcfg repository from your PC -./deploy.sh -``` - -Reboot. - -Check remote unlocking works: `ssh -p 222 root@` - -## Configure wireguard - -```bash -# On node being installed -mkdir -p /var/lib/deuxfleurs/wireguard-keys -cd /var/lib/deuxfleurs/wireguard-keys -wg genkey | tee private | wg pubkey > public -``` - -Get the public key, make sure it is in `cluster.nix` so that nodes know one -another. Also put it anywhere else like in your local wireguard config for -instance so that you can access the node from your PC by its wireguard address -and not only its LAN address. - -Redo a deploy (`./deploy.sh `) - -Check VPN works. Change IP in `ssh_config` to use VPN IP instead of LAN IP (required for deploy when away from home). - -## Commit changes to `nixcfg` repo - -This is a good point to commit your new/modified `.nix` files. - -## Configure Nomad and Consul TLS - -If you are bootstraping a new cluster, you need to `./genpki.sh ` to -make a TLS PKI for the Nomad+Consul cluster to work. Then redo a deploy. diff --git a/experimental/app/csi-s3/deploy/csi-s3.hcl b/experimental/app/csi-s3/deploy/csi-s3.hcl new file mode 100644 index 0000000..8e70c6a --- /dev/null +++ b/experimental/app/csi-s3/deploy/csi-s3.hcl @@ -0,0 +1,39 @@ +job "plugin-csi-s3-nodes" { + datacenters = ["neptune", "pluton"] + + # you can run node plugins as service jobs as well, but this ensures + # that all nodes in the DC have a copy. + type = "system" + + group "nodes" { + task "plugin" { + driver = "docker" + + config { + image = "ctrox/csi-s3:v1.2.0-rc.1" + + args = [ + "--endpoint=unix://csi/csi.sock", + "--nodeid=${node.unique.id}", + "--logtostderr", + "--v=5", + ] + + # node plugins must run as privileged jobs because they + # mount disks to the host + privileged = true + } + + csi_plugin { + id = "csi-s3" + type = "node" + mount_dir = "/csi" + } + + resources { + cpu = 500 + memory = 256 + } + } + } +} diff --git a/experimental/app/csi-s3/deploy/dummy-volume.hcl b/experimental/app/csi-s3/deploy/dummy-volume.hcl new file mode 100644 index 0000000..67dfd39 --- /dev/null +++ b/experimental/app/csi-s3/deploy/dummy-volume.hcl @@ -0,0 +1,20 @@ +id = "dummy-volume" +name = "dummy-volume" +type = "csi" +plugin_id = "csi-s3" + +capability { + access_mode = "single-node-writer" + attachment_mode = "file-system" +} + +secrets { + accessKeyId = "GKfd94f06139bb73de5642baf5" + secretAccessKey = "a4fa6c956d847b145a823c4615e4655126c67babf3cce2337b4d73cd381d7f06" + endpoint = "https://garage-staging.home.adnab.me" + region = "garage-staging" +} + +parameters { + mounter = "rclone" +} diff --git a/experimental/app/nextcloud/config/litestream.yml b/experimental/app/nextcloud/config/litestream.yml new file mode 100644 index 0000000..46eca93 --- /dev/null +++ b/experimental/app/nextcloud/config/litestream.yml @@ -0,0 +1,10 @@ +dbs: + - path: /ephemeral/nextcloud.db + replicas: + - url: s3://nextcloud-db/nextcloud.db + region: garage-staging + endpoint: http://{{ env "attr.unique.network.ip-address" }}:3990 + access-key-id: {{ key "secrets/nextcloud/s3_access_key" | trimSpace }} + secret-access-key: {{ key "secrets/nextcloud/s3_secret_key" | trimSpace }} + force-path-style: true + sync-interval: 60s diff --git a/experimental/app/nextcloud/deploy/nextcloud.hcl b/experimental/app/nextcloud/deploy/nextcloud.hcl new file mode 100644 index 0000000..45d1b6e --- /dev/null +++ b/experimental/app/nextcloud/deploy/nextcloud.hcl @@ -0,0 +1,137 @@ +job "nextcloud" { + datacenters = ["neptune"] + type = "service" + + group "nextcloud" { + count = 1 + + network { + port "http" { + to = 80 + } + } + + ephemeral_disk { + size = 10000 + } + + restart { + attempts = 10 + delay = "30s" + } + + task "restore-db" { + lifecycle { + hook = "prestart" + sidecar = false + } + + driver = "docker" + config { + image = "litestream/litestream:0.3.7" + args = [ + "restore", "-config", "/etc/litestream.yml", "/ephemeral/nextcloud.db" + ] + volumes = [ + "../alloc/data:/ephemeral", + "secrets/litestream.yml:/etc/litestream.yml" + ] + } + user = "33" + + template { + data = file("../config/litestream.yml") + destination = "secrets/litestream.yml" + } + + resources { + memory = 200 + cpu = 1000 + } + } + + task "nextcloud" { + driver = "docker" + config { + image = "nextcloud:22.2.3-apache" + ports = [ "http" ] + #entrypoint = [ "/bin/sh", "-c" ] + #command = "apache2-foreground" + + volumes = [ + "../alloc/data:/var/www/html/data", + ] + } + user = "33" + + template { + data = < +``` + +If necessary, assign static IP. E.g. `ip addr add 192.168.1.40/24 dev eno1` or sth (replace ip and device appropriately) + +Remotely: `ssh-copy-id @`. Check SSH access is good. + +## Deploy from this repo + +See the documentation in `/doc` in this repo. The old procedure described here is partly obsolete. + +## Old guide + +It's time! + +**Files in this repo to create/change:** + +- create node `.nix` file and symlink for node `.site.nix` (create site and + cluster `.nix` files if necessary; use existing files of e.g. the staging + cluster as examples/templates) +- make sure values are filled in correctly +- add node to `ssh_config` with it's LAN IP, we don't have VPN at this stage + +**Configuration steps on the node:** + +```bash +# On node being installed +mkdir -p /var/lib/deuxfleurs/remote-unlock +cd /var/lib/deuxfleurs/remote-unlock +ssh-keygen -t ed25519 -N "" -f ./ssh_host_ed25519_key +``` + +**Try to deploy:** + +```bash +# In nixcfg repository from your PC +./deploy.sh +``` + +Reboot. + +Check remote unlocking works: `ssh -p 222 root@` + +### Configure wireguard + +```bash +# On node being installed +mkdir -p /var/lib/deuxfleurs/wireguard-keys +cd /var/lib/deuxfleurs/wireguard-keys +wg genkey | tee private | wg pubkey > public +``` + +Get the public key, make sure it is in `cluster.nix` so that nodes know one +another. Also put it anywhere else like in your local wireguard config for +instance so that you can access the node from your PC by its wireguard address +and not only its LAN address. + +Redo a deploy (`./deploy.sh `) + +Check VPN works. Change IP in `ssh_config` to use VPN IP instead of LAN IP (required for deploy when away from home). + +### Commit changes to `nixcfg` repo + +This is a good point to commit your new/modified `.nix` files. + +### Configure Nomad and Consul TLS + +If you are bootstraping a new cluster, you need to `./genpki.sh ` to +make a TLS PKI for the Nomad+Consul cluster to work. Then redo a deploy. diff --git a/secretmgr/secretmgr b/secretmgr/secretmgr new file mode 100755 index 0000000..3c0ec08 --- /dev/null +++ b/secretmgr/secretmgr @@ -0,0 +1,381 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i python3 -p "python3.withPackages(ps: [ ps.pip ps.consul ps.ldap ps.passlib ps.requests ps.six ])" + +# DEPENDENCY: python-consul +import consul + +# DEPENDENCY: python-ldap +import ldap + +# DEPENDENCY: passlib +from passlib.hash import ldap_salted_sha1 + +import os +import sys +import glob +import subprocess +import getpass +import base64 +from secrets import token_bytes + + +""" +TODO: this will be a utility to handle secrets in the Consul database +for the various components of the Deuxfleurs infrastructure + +Functionnalities: +- check that secrets are correctly configured +- help user fill in secrets +- create LDAP service users and fill in corresponding secrets +- maybe one day: manage SSL certificates and keys + +It uses files placed in /secrets/* to know what secrets +it should handle. These secret files contain directives for what to do +about these secrets. + +Example directives: + +USER +(a secret that must be filled in by the user) + +USER_LONG +(the same, indicates that the secret fits on several lines) + +CMD +(a secret that is generated by running this command) + +CMD_ONCE +(same, but value is not changed when doing a regen) + +CONST +(the secret has a constant value set here) + +CONST_LONG + +(same) + +SERVICE_DN +(the LDAP DN of a service user) + +SERVICE_PASSWORD +(the LDAP password for the corresponding service user) + +SSL_CERT +(a SSL domain for the given domains) + +SSL_KEY +(the SSL key going with corresponding certificate) + +RSA_PUBLIC_KEY +(a public RSA key) + +RSA_PRIVATE_KEY +(the corresponding private RSA key) +""" + + +# Parameters +LDAP_URL = "ldap://localhost:1389" +SERVICE_DN_SUFFIX = "ou=services,ou=users,dc=deuxfleurs,dc=fr" +consul_server = consul.Consul() + + +# ---- + +USER = "USER" +USER_LONG = "USER_LONG" +CMD = "CMD" +CMD_ONCE = "CMD_ONCE" +CONST = "CONST" +CONST_LONG = "CONST_LONG" +SERVICE_DN = "SERVICE_DN" +SERVICE_PASSWORD = "SERVICE_PASSWORD" +SSL_CERT = "SSL_CERT" +SSL_KEY = "SSL_KEY" +RSA_PUBLIC_KEY = "RSA_PUBLIC_KEY" +RSA_PRIVATE_KEY = "RSA_PRIVATE_KEY" + +class bcolors: + HEADER = '\033[95m' + OKBLUE = '\033[94m' + OKCYAN = '\033[96m' + OKGREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + +def read_secret(key, file_path): + lines = [l.strip() for l in open(file_path, "r")] + if len(lines) == 0: + print(bcolors.FAIL, "ERROR:", bcolors.ENDC, "Empty file in", file_path) + sys.exit(-1) + l0 = lines[0].split(" ") + stype = l0[0] + secret = {"type": stype, "key": key} + if stype in [USER, USER_LONG]: + secret["desc"] = " ".join(l0[1:]) + elif stype in [CMD, CMD_ONCE]: + secret["cmd"] = " ".join(l0[1:]) + elif stype == CONST: + secret["value"] = " ".join(l0[1:]) + elif stype == CONST_LONG: + secret["value"] = "\n".join(lines[1:]) + elif stype in [SERVICE_DN, SERVICE_PASSWORD]: + secret["service"] = l0[1] + if stype == SERVICE_DN: + secret["service_desc"] = " ".join(l0[2:]) + elif stype in [SSL_CERT, SSL_KEY]: + secret["cert_name"] = l0[1] + if stype == SSL_CERT: + secret["cert_domains"] = l0[2:] + elif stype in [RSA_PUBLIC_KEY, RSA_PRIVATE_KEY]: + secret["key_name"] = l0[1] + if stype == RSA_PUBLIC_KEY: + secret["key_desc"] = " ".join(l0[2:]) + else: + print(bcolors.FAIL, "ERROR:", bcolors.ENDC, "Invalid secret type", stype, "in", file_path) + sys.exit(-1) + + return secret + +def read_secrets(module_list): + secrets = {} + for mod in module_list: + for file_path in glob.glob(mod.strip('/') + "/secrets/**", recursive=True): + if os.path.isfile(file_path): + key = '/'.join(file_path.split("/")[1:]) + secrets[key] = read_secret(key, file_path) + return secrets + +def get_secrets_services(secrets): + services = {} + for key, secret in secrets.items(): + if secret["type"] not in [SERVICE_DN, SERVICE_PASSWORD]: + continue + svc = secret["service"] + print(svc, "@", key, bcolors.OKCYAN, "...", bcolors.ENDC) + if svc not in services: + services[svc] = { + "dn": "cn=%s,%s"%(svc, SERVICE_DN_SUFFIX), + "desc": "(not provided)", + "pass": None, + "dn_at": [], + "pass_at": [], + } + if secret["type"] == SERVICE_DN: + services[svc]["dn_at"].append(key) + services[svc]["desc"] = secret["service_desc"] + + if secret["type"] == SERVICE_PASSWORD: + services[svc]["pass_at"].append(key) + _, data = consul_server.kv.get(key) + if data is not None: + if services[svc]["pass"] is None: + services[svc]["pass"] = data["Value"].decode('ascii').strip() + + return services + +ldap_admin_conn = None +def get_ldap_admin_conn(): + global ldap_admin_conn + if ldap_admin_conn is None: + ldap_admin_conn = ldap.initialize(LDAP_URL) + ldap_user = input("LDAP admin user (full DN, please!): ") + ldap_pass = getpass.getpass("LDAP admin password: ") + ldap_admin_conn.simple_bind_s(ldap_user, ldap_pass) + return ldap_admin_conn + +# ---- CHECK COMMAND ---- + +def check_secrets(module_list): + secrets = read_secrets(module_list) + print("Found", len(secrets), "secrets to check") + print() + + check_secrets_presence(secrets) + check_secrets_services(secrets) + +def check_secrets_presence(secrets): + print("Checking secrets presence...") + for key in secrets.keys(): + _, data = consul_server.kv.get(key) + if data is None: + print(key, bcolors.FAIL, "x", bcolors.ENDC) + else: + print(key, bcolors.OKGREEN, "✓", bcolors.ENDC) + print() + +def check_secrets_services(secrets): + print("Checking secrets for LDAP service users...") + services = get_secrets_services(secrets) + + for svc_name, svc in services.items(): + for dn_key in svc["dn_at"]: + _, data = consul_server.kv.get(dn_key) + if data is not None: + got_val = data["Value"].decode('ascii').strip() + if got_val != svc["dn"]: + print(svc_name, "wrong DN at", dn_key, bcolors.FAIL, "x", bcolors.ENDC) + print("got:", got_val, "instead of:", svc["dn"]) + + if svc["pass"] is None: + print(svc_name, bcolors.FAIL, "no password stored", bcolors.ENDC) + else: + for pass_key in svc["pass_at"]: + _, data = consul_server.kv.get(pass_key) + if data is not None: + got_val = data["Value"].decode('ascii').strip() + if got_val != svc["pass"]: + print(svc_name, "wrong pass at", dn_key, bcolors.FAIL, "x", bcolors.ENDC) + + l = ldap.initialize(LDAP_URL) + try: + l.simple_bind_s(svc["dn"], svc["pass"]) + print(svc_name, bcolors.OKGREEN, "✓", bcolors.ENDC) + except Exception as e: + print(svc_name, bcolors.FAIL, e, bcolors.ENDC) + print() + + +# ---- GEN COMMAND ---- + +def gen_secrets(module_list, regen): + secrets = read_secrets(module_list) + print("Found", len(secrets), "secrets to check and maybe generate") + print() + + gen_secrets_base(secrets, regen) + gen_secrets_services(secrets, regen) + + check_secrets_presence(secrets) + check_secrets_services(secrets) + +def gen_secrets_base(secrets, regen): + print("Filling in user secrets and cmd secrets...") + + for key, secret in secrets.items(): + _, data = consul_server.kv.get(key) + if data is not None and not regen: + continue + + if secret["type"] == USER: + print("----") + print(key) + print("Description:", secret["desc"]) + print("Enter value for secret, or ^C to skip:") + try: + val = input().strip() + consul_server.kv.put(key, val) + print(bcolors.OKCYAN, "Value set.", bcolors.ENDC) + except KeyboardInterrupt: + print(bcolors.WARNING, "Skipped.", bcolors.ENDC) + + if secret["type"] == USER_LONG: + print("----") + print(key) + print("Description:", secret["desc"]) + print("Enter value for secret, or ^C to skip:") + print("THIS IS A LONG VALUE, ENTER SEVERAL LINES AND FINISH WITH A LINE CONTAINING A SINGLE .") + try: + lines = [] + while True: + line = input().strip() + if line == ".": + break + lines.append(line) + val = "\n".join(lines) + consul_server.kv.put(key, val) + print(bcolors.OKCYAN, "Value set.", bcolors.ENDC) + except KeyboardInterrupt: + print(bcolors.WARNING, "Skipped.", bcolors.ENDC) + + if secret["type"] in [CONST, CONST_LONG]: + print("----") + print(key) + print("Resetting to constant value.") + consul_server.kv.put(key, secret["value"]) + print(bcolors.OKCYAN, "Value set.", bcolors.ENDC) + + if secret["type"] == CMD or (secret["type"] == CMD_ONCE and data is None): + print("----") + print(key) + print("Executing command:", secret["cmd"]) + val = subprocess.check_output(["sh", "-c", secret["cmd"]]) + consul_server.kv.put(key, val) + print(bcolors.OKCYAN, "Value set.", bcolors.ENDC) + + print() + +def gen_secrets_services(secrets, regen): + print("Generating LDAP service accounts...") + services = get_secrets_services(secrets) + + for svc_name, svc in services.items(): + print("----") + print("Service:", svc_name) + print("Description:", svc["desc"]) + + for dn_key in svc["dn_at"]: + _, data = consul_server.kv.get(dn_key) + if data is None or data["Value"].decode('ascii').strip() != svc["dn"]: + print(bcolors.OKCYAN, "Setting DN", bcolors.ENDC, "at", dn_key) + consul_server.kv.put(dn_key, svc["dn"]) + + if svc["pass"] is None or regen: + print(bcolors.OKCYAN, "Generating new password", bcolors.ENDC) + svc["pass"] = base64.urlsafe_b64encode(token_bytes(12)).decode('ascii') + + l = ldap.initialize(LDAP_URL) + try: + l.simple_bind_s(svc["dn"], svc["pass"]) + except: + fix_service_user(svc) + + for pass_key in svc["pass_at"]: + _, data = consul_server.kv.get(pass_key) + if data is None or data["Value"].decode('ascii').strip() != svc["pass"]: + print(bcolors.OKCYAN, "Setting password", bcolors.ENDC, "at", pass_key) + consul_server.kv.put(pass_key, svc["pass"]) + + print() + +def fix_service_user(svc): + print("Fixing service user", svc["dn"], "...") + l = get_ldap_admin_conn() + res = l.search_s(svc["dn"], ldap.SCOPE_BASE, "objectclass=*") + pass_crypt = ldap_salted_sha1.hash(svc["pass"]) + if res is None or len(res) == 0: + print(bcolors.OKCYAN, "Creating entity...", bcolors.ENDC) + l.add_s(svc["dn"], + [ + ("objectclass", [b"person", b"top"]), + ("displayname", [svc["desc"].encode('ascii')]), + ("userpassword", [pass_crypt.encode('ascii')]), + ]) + else: + print(bcolors.OKCYAN, "Resetting entity password", bcolors.ENDC) + l.modify_s(svc["dn"], + [ + (ldap.MOD_REPLACE, "userpassword", [pass_crypt.encode('ascii')]) + ]) + +# ---- MAIN ---- + +if __name__ == "__main__": + for i, val in enumerate(sys.argv): + if val == "check": + check_secrets(sys.argv[i+1:]) + break + elif val == "gen": + gen_secrets(sys.argv[i+1:], False) + break + elif val == "regen": + gen_secrets(sys.argv[i+1:], True) + break + else: + print("Usage:") + print(" secretmgr.py [check|gen|regen] ...") + + diff --git a/secretmgr/secretmgr.py b/secretmgr/secretmgr.py deleted file mode 100755 index 8b17f61..0000000 --- a/secretmgr/secretmgr.py +++ /dev/null @@ -1,380 +0,0 @@ -#!/usr/bin/env python3 - -# DEPENDENCY: python-consul -import consul - -# DEPENDENCY: python-ldap -import ldap - -# DEPENDENCY: passlib -from passlib.hash import ldap_salted_sha1 - -import os -import sys -import glob -import subprocess -import getpass -import base64 -from secrets import token_bytes - - -""" -TODO: this will be a utility to handle secrets in the Consul database -for the various components of the Deuxfleurs infrastructure - -Functionnalities: -- check that secrets are correctly configured -- help user fill in secrets -- create LDAP service users and fill in corresponding secrets -- maybe one day: manage SSL certificates and keys - -It uses files placed in /secrets/* to know what secrets -it should handle. These secret files contain directives for what to do -about these secrets. - -Example directives: - -USER -(a secret that must be filled in by the user) - -USER_LONG -(the same, indicates that the secret fits on several lines) - -CMD -(a secret that is generated by running this command) - -CMD_ONCE -(same, but value is not changed when doing a regen) - -CONST -(the secret has a constant value set here) - -CONST_LONG - -(same) - -SERVICE_DN -(the LDAP DN of a service user) - -SERVICE_PASSWORD -(the LDAP password for the corresponding service user) - -SSL_CERT -(a SSL domain for the given domains) - -SSL_KEY -(the SSL key going with corresponding certificate) - -RSA_PUBLIC_KEY -(a public RSA key) - -RSA_PRIVATE_KEY -(the corresponding private RSA key) -""" - - -# Parameters -LDAP_URL = "ldap://localhost:1389" -SERVICE_DN_SUFFIX = "ou=services,ou=users,dc=deuxfleurs,dc=fr" -consul_server = consul.Consul() - - -# ---- - -USER = "USER" -USER_LONG = "USER_LONG" -CMD = "CMD" -CMD_ONCE = "CMD_ONCE" -CONST = "CONST" -CONST_LONG = "CONST_LONG" -SERVICE_DN = "SERVICE_DN" -SERVICE_PASSWORD = "SERVICE_PASSWORD" -SSL_CERT = "SSL_CERT" -SSL_KEY = "SSL_KEY" -RSA_PUBLIC_KEY = "RSA_PUBLIC_KEY" -RSA_PRIVATE_KEY = "RSA_PRIVATE_KEY" - -class bcolors: - HEADER = '\033[95m' - OKBLUE = '\033[94m' - OKCYAN = '\033[96m' - OKGREEN = '\033[92m' - WARNING = '\033[93m' - FAIL = '\033[91m' - ENDC = '\033[0m' - BOLD = '\033[1m' - UNDERLINE = '\033[4m' - -def read_secret(key, file_path): - lines = [l.strip() for l in open(file_path, "r")] - if len(lines) == 0: - print(bcolors.FAIL, "ERROR:", bcolors.ENDC, "Empty file in", file_path) - sys.exit(-1) - l0 = lines[0].split(" ") - stype = l0[0] - secret = {"type": stype, "key": key} - if stype in [USER, USER_LONG]: - secret["desc"] = " ".join(l0[1:]) - elif stype in [CMD, CMD_ONCE]: - secret["cmd"] = " ".join(l0[1:]) - elif stype == CONST: - secret["value"] = " ".join(l0[1:]) - elif stype == CONST_LONG: - secret["value"] = "\n".join(lines[1:]) - elif stype in [SERVICE_DN, SERVICE_PASSWORD]: - secret["service"] = l0[1] - if stype == SERVICE_DN: - secret["service_desc"] = " ".join(l0[2:]) - elif stype in [SSL_CERT, SSL_KEY]: - secret["cert_name"] = l0[1] - if stype == SSL_CERT: - secret["cert_domains"] = l0[2:] - elif stype in [RSA_PUBLIC_KEY, RSA_PRIVATE_KEY]: - secret["key_name"] = l0[1] - if stype == RSA_PUBLIC_KEY: - secret["key_desc"] = " ".join(l0[2:]) - else: - print(bcolors.FAIL, "ERROR:", bcolors.ENDC, "Invalid secret type", stype, "in", file_path) - sys.exit(-1) - - return secret - -def read_secrets(module_list): - secrets = {} - for mod in module_list: - for file_path in glob.glob(mod.strip('/') + "/secrets/**", recursive=True): - if os.path.isfile(file_path): - key = '/'.join(file_path.split("/")[1:]) - secrets[key] = read_secret(key, file_path) - return secrets - -def get_secrets_services(secrets): - services = {} - for key, secret in secrets.items(): - if secret["type"] not in [SERVICE_DN, SERVICE_PASSWORD]: - continue - svc = secret["service"] - print(svc, "@", key, bcolors.OKCYAN, "...", bcolors.ENDC) - if svc not in services: - services[svc] = { - "dn": "cn=%s,%s"%(svc, SERVICE_DN_SUFFIX), - "desc": "(not provided)", - "pass": None, - "dn_at": [], - "pass_at": [], - } - if secret["type"] == SERVICE_DN: - services[svc]["dn_at"].append(key) - services[svc]["desc"] = secret["service_desc"] - - if secret["type"] == SERVICE_PASSWORD: - services[svc]["pass_at"].append(key) - _, data = consul_server.kv.get(key) - if data is not None: - if services[svc]["pass"] is None: - services[svc]["pass"] = data["Value"].decode('ascii').strip() - - return services - -ldap_admin_conn = None -def get_ldap_admin_conn(): - global ldap_admin_conn - if ldap_admin_conn is None: - ldap_admin_conn = ldap.initialize(LDAP_URL) - ldap_user = input("LDAP admin user (full DN, please!): ") - ldap_pass = getpass.getpass("LDAP admin password: ") - ldap_admin_conn.simple_bind_s(ldap_user, ldap_pass) - return ldap_admin_conn - -# ---- CHECK COMMAND ---- - -def check_secrets(module_list): - secrets = read_secrets(module_list) - print("Found", len(secrets), "secrets to check") - print() - - check_secrets_presence(secrets) - check_secrets_services(secrets) - -def check_secrets_presence(secrets): - print("Checking secrets presence...") - for key in secrets.keys(): - _, data = consul_server.kv.get(key) - if data is None: - print(key, bcolors.FAIL, "x", bcolors.ENDC) - else: - print(key, bcolors.OKGREEN, "✓", bcolors.ENDC) - print() - -def check_secrets_services(secrets): - print("Checking secrets for LDAP service users...") - services = get_secrets_services(secrets) - - for svc_name, svc in services.items(): - for dn_key in svc["dn_at"]: - _, data = consul_server.kv.get(dn_key) - if data is not None: - got_val = data["Value"].decode('ascii').strip() - if got_val != svc["dn"]: - print(svc_name, "wrong DN at", dn_key, bcolors.FAIL, "x", bcolors.ENDC) - print("got:", got_val, "instead of:", svc["dn"]) - - if svc["pass"] is None: - print(svc_name, bcolors.FAIL, "no password stored", bcolors.ENDC) - else: - for pass_key in svc["pass_at"]: - _, data = consul_server.kv.get(pass_key) - if data is not None: - got_val = data["Value"].decode('ascii').strip() - if got_val != svc["pass"]: - print(svc_name, "wrong pass at", dn_key, bcolors.FAIL, "x", bcolors.ENDC) - - l = ldap.initialize(LDAP_URL) - try: - l.simple_bind_s(svc["dn"], svc["pass"]) - print(svc_name, bcolors.OKGREEN, "✓", bcolors.ENDC) - except Exception as e: - print(svc_name, bcolors.FAIL, e, bcolors.ENDC) - print() - - -# ---- GEN COMMAND ---- - -def gen_secrets(module_list, regen): - secrets = read_secrets(module_list) - print("Found", len(secrets), "secrets to check and maybe generate") - print() - - gen_secrets_base(secrets, regen) - gen_secrets_services(secrets, regen) - - check_secrets_presence(secrets) - check_secrets_services(secrets) - -def gen_secrets_base(secrets, regen): - print("Filling in user secrets and cmd secrets...") - - for key, secret in secrets.items(): - _, data = consul_server.kv.get(key) - if data is not None and not regen: - continue - - if secret["type"] == USER: - print("----") - print(key) - print("Description:", secret["desc"]) - print("Enter value for secret, or ^C to skip:") - try: - val = input().strip() - consul_server.kv.put(key, val) - print(bcolors.OKCYAN, "Value set.", bcolors.ENDC) - except KeyboardInterrupt: - print(bcolors.WARNING, "Skipped.", bcolors.ENDC) - - if secret["type"] == USER_LONG: - print("----") - print(key) - print("Description:", secret["desc"]) - print("Enter value for secret, or ^C to skip:") - print("THIS IS A LONG VALUE, ENTER SEVERAL LINES AND FINISH WITH A LINE CONTAINING A SINGLE .") - try: - lines = [] - while True: - line = input().strip() - if line == ".": - break - lines.append(line) - val = "\n".join(lines) - consul_server.kv.put(key, val) - print(bcolors.OKCYAN, "Value set.", bcolors.ENDC) - except KeyboardInterrupt: - print(bcolors.WARNING, "Skipped.", bcolors.ENDC) - - if secret["type"] in [CONST, CONST_LONG]: - print("----") - print(key) - print("Resetting to constant value.") - consul_server.kv.put(key, secret["value"]) - print(bcolors.OKCYAN, "Value set.", bcolors.ENDC) - - if secret["type"] == CMD or (secret["type"] == CMD_ONCE and data is None): - print("----") - print(key) - print("Executing command:", secret["cmd"]) - val = subprocess.check_output(["sh", "-c", secret["cmd"]]) - consul_server.kv.put(key, val) - print(bcolors.OKCYAN, "Value set.", bcolors.ENDC) - - print() - -def gen_secrets_services(secrets, regen): - print("Generating LDAP service accounts...") - services = get_secrets_services(secrets) - - for svc_name, svc in services.items(): - print("----") - print("Service:", svc_name) - print("Description:", svc["desc"]) - - for dn_key in svc["dn_at"]: - _, data = consul_server.kv.get(dn_key) - if data is None or data["Value"].decode('ascii').strip() != svc["dn"]: - print(bcolors.OKCYAN, "Setting DN", bcolors.ENDC, "at", dn_key) - consul_server.kv.put(dn_key, svc["dn"]) - - if svc["pass"] is None or regen: - print(bcolors.OKCYAN, "Generating new password", bcolors.ENDC) - svc["pass"] = base64.urlsafe_b64encode(token_bytes(12)).decode('ascii') - - l = ldap.initialize(LDAP_URL) - try: - l.simple_bind_s(svc["dn"], svc["pass"]) - except: - fix_service_user(svc) - - for pass_key in svc["pass_at"]: - _, data = consul_server.kv.get(pass_key) - if data is None or data["Value"].decode('ascii').strip() != svc["pass"]: - print(bcolors.OKCYAN, "Setting password", bcolors.ENDC, "at", pass_key) - consul_server.kv.put(pass_key, svc["pass"]) - - print() - -def fix_service_user(svc): - print("Fixing service user", svc["dn"], "...") - l = get_ldap_admin_conn() - res = l.search_s(svc["dn"], ldap.SCOPE_BASE, "objectclass=*") - pass_crypt = ldap_salted_sha1.hash(svc["pass"]) - if res is None or len(res) == 0: - print(bcolors.OKCYAN, "Creating entity...", bcolors.ENDC) - l.add_s(svc["dn"], - [ - ("objectclass", [b"person", b"top"]), - ("displayname", [svc["desc"].encode('ascii')]), - ("userpassword", [pass_crypt.encode('ascii')]), - ]) - else: - print(bcolors.OKCYAN, "Resetting entity password", bcolors.ENDC) - l.modify_s(svc["dn"], - [ - (ldap.MOD_REPLACE, "userpassword", [pass_crypt.encode('ascii')]) - ]) - -# ---- MAIN ---- - -if __name__ == "__main__": - for i, val in enumerate(sys.argv): - if val == "check": - check_secrets(sys.argv[i+1:]) - break - elif val == "gen": - gen_secrets(sys.argv[i+1:], False) - break - elif val == "regen": - gen_secrets(sys.argv[i+1:], True) - break - else: - print("Usage:") - print(" secretmgr.py [check|gen|regen] ...") - - diff --git a/secretmgr/shell.nix b/secretmgr/shell.nix deleted file mode 100644 index c9b8053..0000000 --- a/secretmgr/shell.nix +++ /dev/null @@ -1,15 +0,0 @@ -{ - pkgs ? import {} -}: - -with pkgs; mkShell { - nativeBuildInputs = [ - nomad - docker-compose - python39Packages.pip - python39Packages.ldap - python39Packages.consul - python39Packages.passlib - ]; -} - diff --git a/sshtool b/sshtool index 262f0e3..24c19af 100755 --- a/sshtool +++ b/sshtool @@ -1,6 +1,11 @@ #!/usr/bin/env bash CMDFILE="$1" +if [ -z "$CMDFILE" ] || [ ! -f "$CMDFILE" ]; then + echo "sshtool is not meant to be called on its own." + echo "See scripts that use it (e.g. deploy_nixos) for usage examples." + exit 1 +fi shift 1 cd $(dirname $CMDFILE) -- cgit v1.2.3