From a13c37049a4a9c38ccc3acb0d78ed2717a29244c Mon Sep 17 00:00:00 2001 From: Quentin Date: Wed, 8 Dec 2021 15:43:59 +0100 Subject: Add minio, a README and some small enhancements --- README.md | 43 ++++++++++++++++++++++++++++++++ example/deploy_garage.sh | 8 +++--- example/deploy_minio.py | 64 ++++++++++++++++++++++++++++++++++++++++++++++++ main.py | 6 +++-- 4 files changed, 115 insertions(+), 6 deletions(-) create mode 100644 README.md create mode 100755 example/deploy_minio.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..270d616 --- /dev/null +++ b/README.md @@ -0,0 +1,43 @@ +# mknet + +Show usage: + +```help +python3 main.py +``` + +## Example + +```bash +sudo rm -rf /tmp/garage-testnet/ # always start by deleting previous run +sudo python3 main.py create ./config.yml +sudo python3 main.py run-all example/deploy_garage.sh +sudo python3 main.py run dc1:dc1s1 garage -c /tmp/garage-testnet/dc1/dc1s1/garage.toml status +sudo python3 main.py destroy +``` + +## Instrumented daemons + +If you want to use the scripts provided in the `example` folder, +you must add to your path some tools. + +### Garage (`deploy_garage.sh`) + +```bash +# see versions on https://garagehq.deuxfleurs.fr/_releases.html +export GRG_ARCH=x86_64-unknown-linux-musl +export GRG_VERSION=v0.5.0 + +sudo wget https://garagehq.deuxfleurs.fr/_releases/${GRG_VERSION}/${GRG_ARCH}/garage -O /usr/local/bin/garage +sudo chmod +x /usr/local/bin/garage + +garage help +``` + +### Minio (`deploy_minio.py`) + +``` +sudo wget https://dl.min.io/server/minio/release/linux-amd64/minio -O /usr/local/bin/minio +sudo chmod +x /usr/local/bin/minio +``` + diff --git a/example/deploy_garage.sh b/example/deploy_garage.sh index e91b387..0dc9b38 100755 --- a/example/deploy_garage.sh +++ b/example/deploy_garage.sh @@ -46,15 +46,15 @@ EOF RUST_LOG=garage=debug ${GARAGE_PATH} server 2>> ${NODE_STORAGE_PATH}/logs & disown sleep 2 -CONFIG_NODE_FPATH=(${STORAGE_PATH}/*{,/*}/garage.toml) +CONFIG_NODE_FPATH=$(find /tmp/garage-testnet/ -maxdepth 3 -name garage.toml|head -n 1) SELF_ID=$(${GARAGE_PATH} node id 2>/dev/null) SHORT_ID=$(echo ${SELF_ID} | cut -c-64) -${GARAGE_PATH} -c ${CONFIG_NODE_FPATH[0]} node connect ${SELF_ID} -${GARAGE_PATH} -c ${CONFIG_NODE_FPATH[0]} layout assign ${SHORT_ID} -z ${ZONE:-unzonned-${HOST}} -c 1 -t ${HOST} +${GARAGE_PATH} -c ${CONFIG_NODE_FPATH} node connect ${SELF_ID} +${GARAGE_PATH} -c ${CONFIG_NODE_FPATH} layout assign ${SHORT_ID} -z ${ZONE:-unzonned-${HOST}} -c 1 -t ${HOST} -if [ ${CONFIG_NODE_FPATH[0]} == ${GARAGE_CONFIG_FILE} ]; then +if [ ${CONFIG_NODE_FPATH} == ${GARAGE_CONFIG_FILE} ]; then sleep 2 ${GARAGE_PATH} layout show ${GARAGE_PATH} layout apply --version 1 diff --git a/example/deploy_minio.py b/example/deploy_minio.py new file mode 100755 index 0000000..8ee8430 --- /dev/null +++ b/example/deploy_minio.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +import json, os, sys, time, pathlib, socket, shutil + +STORAGE_PATH = os.path.join(os.getcwd(), '.minio-testnet') +MINIO_PATH = '/srv' +HOSTS_PATH = os.path.join(STORAGE_PATH, 'hosts.txt') +UNIX_SOCK = os.path.join(STORAGE_PATH, 'deploy.sock') +DATA_PATH = lambda nid: os.path.join(STORAGE_PATH, 'data'+str(nid)) + +def main(): + if int(os.environ['ID']) == 1: leader() + else: follower() + +def leader(): + shutil.rmtree(STORAGE_PATH, ignore_errors=True) + os.makedirs(STORAGE_PATH) + os.makedirs(MINIO_PATH, exist_ok=True) + print(STORAGE_PATH) + + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.bind(UNIX_SOCK) + sock.listen() + + n_serv = os.environ['SIZE'] + fl = [ co for co, addr in [ sock.accept() for i in range(n_serv - 1) ]] + + identities = [ json.loads(co.makefile().readline()) for co in fl ] + [ { "ip": os.environ['IP'], "path": make_data() } ] + print(f"ident: {identities}") + msg = f"{json.dumps(identities)}\n".encode() + [ co.send(msg) for co in fl ] + + run_minio(identities) + +def follower(): + co = None + while True: + time.sleep(1) + try: + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.connect(UNIX_SOCK) + co = sock.makefile() + break + except Exception as err: + print('conn failed, wait,', err) + my_identity = json.dumps({ "ip": os.environ['IP'], "path": make_data() }) + sock.send(f"{my_identity}\n".encode()) + identities = json.loads(co.readline()) + + run_minio(identities) + +def make_data(): + data_path = DATA_PATH(os.environ['ID']) + os.makedirs(data_path) + return data_path + +def run_minio(identities): + cmd = f"minio server --console-address ':9001' --address [{os.environ['IP']}]:9000" + for ident in identities: + cmd += f" http://[{ident['ip']}]:9000{ident['path']}" + cmd += f" > {os.path.join(STORAGE_PATH, 'minio'+os.environ['ID']+'.log')} 2>&1" + print("launch: ", cmd) + os.system(cmd) + +__name__ == '__main__' and main() diff --git a/main.py b/main.py index c243a2f..456da52 100755 --- a/main.py +++ b/main.py @@ -1,4 +1,4 @@ -#!/bin/env python +#!/usr/bin/env python import ipaddress import os import shutil @@ -305,7 +305,7 @@ def run(netns, cmd): if len(cmd) == 0: cmd = [os.getenv("SHELL") or "/bin/sh"] - os.execve("/bin/env", ["/bin/env", "ip", "netns" , "exec", name ] + cmd, env) + os.execve("/usr/bin/env", ["/usr/bin/env", "ip", "netns" , "exec", name ] + cmd, env) def runall(cmd): with open(".current_state.yml", "r") as file: @@ -321,6 +321,7 @@ def runall(cmd): env["HOST"] = server.name env["IP"] = str(server.ip) env["ID"] = str(number) + env["SIZE"] = str(len(config['servers'])) name = f'testnet-{zone.name}-{server.name}' net.ns.run(name, cmd, env) number +=1 @@ -330,6 +331,7 @@ def runall(cmd): env["HOST"] = zone.name env["IP"] = str(zone.ip) env["ID"] = str(number) + env["SIZE"] = str(len(config['servers'])) name = f'testnet-{zone.name}-{zone.name}' net.ns.run(name, cmd, env) first = False -- cgit v1.2.3 From abccfb88b8cc873b4f399d38a3151bbbe12c360d Mon Sep 17 00:00:00 2001 From: Quentin Date: Wed, 8 Dec 2021 16:26:56 +0100 Subject: Make mknet installable via pip --- main.py | 364 -------------------------------------------------------------- mknet | 365 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 6 ++ 3 files changed, 371 insertions(+), 364 deletions(-) delete mode 100755 main.py create mode 100755 mknet create mode 100644 setup.py diff --git a/main.py b/main.py deleted file mode 100755 index 456da52..0000000 --- a/main.py +++ /dev/null @@ -1,364 +0,0 @@ -#!/usr/bin/env python -import ipaddress -import os -import shutil -import subprocess -import sys -import yaml -import net - -class SubnetManager: - def __init__(self, config): - self.base = config['base'] - self.zone_size = config['zone'] - self.local_size = config['local'] - if ipaddress.ip_address(self.base).version == 6: - self.prefix = 128 - self.zone_size - self.local_size - else: - self.prefix = 32 - self.zone_size - self.local_size - self.networks = ipaddress.ip_network((self.base, self.prefix)).subnets(self.zone_size) - - self.current_net = next(self.networks).hosts() - - def next_local(self): - return next(self.current_net) - - def next_zone(self): - self.current_net = next(self.networks).hosts() - - def __str__(self): - return f'SubnetManager{{base: {self.base}, zone: {self.zone_size}, local: {self.local_size}}}' - -class Latency: - def __init__(self, latency, offset = None): - if type(latency) is int: - self.latency_us = latency - else: - for suffix, factor in [("us", 1),("ms", 1000), ("s", 1000000), ("m", 60 * 1000000)]: - if latency.endswith(suffix): - self.latency_us = int(float(latency[:-len(suffix)]) * factor) - break - else: - self.latency_us = int(latency) * 1000 - if offset: - self.latency_us -= Latency(offset).latency_us - if self.latency_us < 0: - self.latency_us = 0 - - def __eq__(self, o): - return isinstance(o, Latency) and o.latency_us == self.latency_us - - def __str__(self): - return f'{self.latency_us}ms' - -class Bandwidth: - def __init__(self, bw): - def convert(bw): - factor = 1 - for suffix, f in [("bit", 1), ("bps",1), ("b", 1), ("byte", 8), ("Byte",8), ("B", 8)]: - if bw.endswith(suffix): - bw = bw[:-len(suffix)] - factor = f - break - - for suffix, f in [("k", 1000), ("ki", 1024), ("m", 1000**2), ("mi", 1024**2), ("g", 1000**3), ("gi", 1024**3)]: - if bw.lower().endswith(suffix): - return int(float(bw[:-len(suffix)]) * factor * f) - else: - return int(float(bw) * factor) - if type(bw) is dict: - self.down = convert(bw["down"]) - self.up = convert(bw["up"]) - else: - self.down = convert(bw) - self.up = convert(bw) - - def __str__(self): - def convert(bw): - for suffix, factor in [("g", 1000**3), ("m", 1000**2), ("k", 1000)]: - if bw > 10 * factor: - return f'{bw/factor:.1f}{suffix}bps' - return f'{convert(self.down)}/{convert(self.up)}' - - def __eq__(self, o): - return (isinstance(o, Bandwidth) and - o.down == self.down and - o.up == self.up) - - -class LinkInfo: - def __init__(self, bandwidth, latency, jitter = None, offset = None, **kwargs): - self.bandwidth = Bandwidth(bandwidth) - self.latency = Latency(latency, offset) - self.jitter = Latency(jitter or 0) - - def __eq__(self, o): - return (isinstance(o, LinkInfo) and - o.bandwidth == self.bandwidth and - o.latency == self.latency and - o.jitter == self.jitter) - - def __str__(self): - return f'LinkInfo{{bw: {self.bandwidth}, latency: {self.latency}, jitter: {self.jitter}}}' - -class Server: - def __init__(self, name, link): - self.ip = None - self.name = name - self.link = link - - def is_zone(self): - return False - - def __str__(self): - return f'Server{{name: {self.name}, ip: {self.ip}, link: {self.link}}}' - - def __repr__(self): - return self.__str__() - -class Zone: - def __init__(self, name): - self.name = name - self.link = None - self.servers = {} - - def is_zone(self): - return True - - def add_server(self, server): - if self.servers.get(server.name) is None: - self.servers[server.name] = server - else: - raise Exception(f"Duplicate server '{server.name}' in zone '{self.name}'") - - def set_link(self, link): - if self.link is None: - self.link = link - elif self.link != link: - raise Exception(f"Uncoherent link configuration for zone '{self.name}'") - - def __str__(self): - return f'Zone{{name: {self.name}, link: {self.link}, servers: {list(self.servers.values())}}}' - - def __repr__(self): - return self.__str__() - - -class Network: - def __init__(self): - self.zones = {} - self.subnet_manager = None - self.latency_off = Latency(0) - self.host_ip = None - self.host_link = None - - def set_subnet_manager(self, subnet): - self.subnet_manager = SubnetManager(subnet) - - def set_latency_offset(self, latency_off): - self.latency_off = latency_off - - def add_server(self, server): - name = server["name"] - if zone_obj := server.get("zone"): - zone_name = zone_obj["name"] - if zone := self.zones.get(zone_name): - if not zone.is_zone(): - raise Exception("Duplicate zone: " + name) - else: - zone = Zone(zone_name) - self.zones[zone_name] = zone - if link:=zone_obj.get("external"): - zone.set_link(LinkInfo(offset = self.latency_off, **link)) - zone.add_server(Server(name, LinkInfo(**zone_obj["internal"]))) - - else: - name = name - if name in self.zones: - raise Exception("Duplicate zone: " + name) - self.zones[name] = Server(name, LinkInfo(offset = self.latency_off, **server)) - - def assign_ips(self): - for zone in self.zones.values(): - if zone.is_zone(): - for server in zone.servers.values(): - server.ip = self.subnet_manager.next_local() - else: - zone.ip = self.subnet_manager.next_local() - self.subnet_manager.next_zone() - if not self.host_ip: - self.host_ip = self.subnet_manager.next_local() - - def __str__(self): - return f'Network{{subnet_manager: {self.subnet_manager}, zones: {list(self.zones.values())}, latency_offset: {self.latency_off}}}' - -class NamespaceManager: - def __init__(self): - self.namespaces = set(["unconfined"]) - self.prefixlen = 0 - net.ns.name_unconfined() - - def make_namespace(self, name): - if not name in self.namespaces: - net.ns.create(name) - self.namespaces.add(name) - - def make_bridge(self, name, namespace, ports): - self.make_namespace(namespace) - net.create_bridge(name, namespace, ports) - - def make_veth(self, name1, name2, space1, space2, ip, link=None): - self.make_namespace(space1) - self.make_namespace(space2) - net.create_veth(name1, space1, name2, space2, ip, self.prefixlen, link) - - def build_network(self, network): - self.prefixlen = network.subnet_manager.prefix - netns = "testnet-core" - self.make_veth("veth-testnet", "unconfined", "unconfined", netns, network.host_ip, network.host_link) - ports = ["unconfined"] - for zone in network.zones.values(): - if zone.is_zone(): - self.build_zone(zone) - else: - self.build_server(zone) - ports.append('veth-' + zone.name) - self.make_bridge("br0", netns, ports) - - def build_zone(self, zone): - netns = "testnet-" + zone.name - self.make_veth("veth-" + zone.name, "veth-" + zone.name, netns, "testnet-core", None, zone.link) - ports = ['veth-' + zone.name] - for server in zone.servers.values(): - self.build_server(server, zone) - ports.append('veth-' + server.name) - self.make_bridge("br-" + zone.name, netns, ports) - - def build_server(self, server, zone = None): - if zone: - zone_name = "testnet-" + zone.name - namespace = zone_name + "-" + server.name - else: - zone_name = "testnet-core" - namespace = "testnet-" + server.name + "-" + server.name - self.make_veth("veth", "veth-" + server.name, namespace, zone_name, server.ip, server.link) - -def parse(yaml): - server_list = yaml["servers"] - global_conf = yaml.get("global", {}) - subnet = global_conf.get("subnet", {'base': 'fc00:9a7a:9e::', 'local': 64, 'zone': 16}) - latency_offset = global_conf.get("latency-offset", 0) - - network = Network() - if upstream := global_conf.get("upstream"): - network.host_ip = upstream.get("ip") - if host_link:= upstream.get("conn"): - network.host_link = LinkInfo(latency_offset=latency_offset, **host_link) - network.set_subnet_manager(subnet) - network.set_latency_offset(latency_offset) - for server in server_list: - network.add_server(server) - network.assign_ips() - return network - -def create(config_path): - with open(config_path, "r") as file: - config = yaml.safe_load(file) - shutil.copy(config_path, ".current_state.yml") - network = parse(config) - nsm = NamespaceManager() - nsm.build_network(network) - -def run(netns, cmd): - if ":" in netns: - zone_name,host = netns.split(":", 1) - else: - zone_name,host = None, netns - with open(".current_state.yml", "r") as file: - config = yaml.safe_load(file) - zones = parse(config).zones - server = None - zone = None - if zone_name: - if (zone := zones.get(zone_name)) and zone.is_zone(): - server = zone.servers.get(host) - elif (zone := zones.get(host)) and not zone.is_zone(): - server = zone - else: - for z in zones.values(): - if not z.is_zone(): continue - if (s := z.servers.get(host)): - if server: - raise Exception("Multiple matching host found.") - server = s - zone = z - - if not server: - raise Exception("No matching host was found") - - env = os.environ.copy() - env["HOST"] = server.name - if zone.is_zone(): - env["ZONE"] = zone.name - env["IP"] = str(server.ip) - name = f'testnet-{zone.name}-{server.name}' - - if len(cmd) == 0: - cmd = [os.getenv("SHELL") or "/bin/sh"] - os.execve("/usr/bin/env", ["/usr/bin/env", "ip", "netns" , "exec", name ] + cmd, env) - -def runall(cmd): - with open(".current_state.yml", "r") as file: - config = yaml.safe_load(file) - zones = parse(config).zones - - number = 1 - for zone in zones.values(): - if zone.is_zone(): - for server in zone.servers.values(): - env = os.environ.copy() - env["ZONE"] = zone.name - env["HOST"] = server.name - env["IP"] = str(server.ip) - env["ID"] = str(number) - env["SIZE"] = str(len(config['servers'])) - name = f'testnet-{zone.name}-{server.name}' - net.ns.run(name, cmd, env) - number +=1 - else: - env = os.environ.copy() - env["ZONE"] = "" - env["HOST"] = zone.name - env["IP"] = str(zone.ip) - env["ID"] = str(number) - env["SIZE"] = str(len(config['servers'])) - name = f'testnet-{zone.name}-{zone.name}' - net.ns.run(name, cmd, env) - first = False - number +=1 - -def destroy(): - for ns in net.ns.list(): - net.ns.kill(ns) - net.ns.forget("unconfined") - os.remove(".current_state.yml") - -if __name__ == "__main__": - if len(sys.argv) < 2: - print("""Usage: - mk-testnet create [config_path] # create a new network. config_path defailt to config.yml - mk-testnet run-all [args...] # run a command as each host. set the IP, NAME and ZONE environment variables - mk-testnet run [cmd [args...]] # run command in host named . Use zonename:name if multiple zones hosts server with same name. If cmd is empty, run a shell - mk-testnet destroy # destroy the current environment""") - exit() - cmd = sys.argv[1] - if cmd == "create": - create(sys.argv[2] if len(sys.argv) > 2 else "config.yml") - elif cmd == "run": - run(sys.argv[2], sys.argv[3:]) - elif cmd == "run-all": - runall(sys.argv[2:]) - elif cmd == "destroy": - destroy() - else: - raise Exception(f"Unknown command: {cmd}") diff --git a/mknet b/mknet new file mode 100755 index 0000000..8eb276a --- /dev/null +++ b/mknet @@ -0,0 +1,365 @@ +#!/usr/bin/env python +import ipaddress +import os +import shutil +import subprocess +import sys +import yaml +import net + +class SubnetManager: + def __init__(self, config): + self.base = config['base'] + self.zone_size = config['zone'] + self.local_size = config['local'] + if ipaddress.ip_address(self.base).version == 6: + self.prefix = 128 - self.zone_size - self.local_size + else: + self.prefix = 32 - self.zone_size - self.local_size + self.networks = ipaddress.ip_network((self.base, self.prefix)).subnets(self.zone_size) + + self.current_net = next(self.networks).hosts() + + def next_local(self): + return next(self.current_net) + + def next_zone(self): + self.current_net = next(self.networks).hosts() + + def __str__(self): + return f'SubnetManager{{base: {self.base}, zone: {self.zone_size}, local: {self.local_size}}}' + +class Latency: + def __init__(self, latency, offset = None): + if type(latency) is int: + self.latency_us = latency + else: + for suffix, factor in [("us", 1),("ms", 1000), ("s", 1000000), ("m", 60 * 1000000)]: + if latency.endswith(suffix): + self.latency_us = int(float(latency[:-len(suffix)]) * factor) + break + else: + self.latency_us = int(latency) * 1000 + if offset: + self.latency_us -= Latency(offset).latency_us + if self.latency_us < 0: + self.latency_us = 0 + + def __eq__(self, o): + return isinstance(o, Latency) and o.latency_us == self.latency_us + + def __str__(self): + return f'{self.latency_us}ms' + +class Bandwidth: + def __init__(self, bw): + def convert(bw): + factor = 1 + for suffix, f in [("bit", 1), ("bps",1), ("b", 1), ("byte", 8), ("Byte",8), ("B", 8)]: + if bw.endswith(suffix): + bw = bw[:-len(suffix)] + factor = f + break + + for suffix, f in [("k", 1000), ("ki", 1024), ("m", 1000**2), ("mi", 1024**2), ("g", 1000**3), ("gi", 1024**3)]: + if bw.lower().endswith(suffix): + return int(float(bw[:-len(suffix)]) * factor * f) + else: + return int(float(bw) * factor) + if type(bw) is dict: + self.down = convert(bw["down"]) + self.up = convert(bw["up"]) + else: + self.down = convert(bw) + self.up = convert(bw) + + def __str__(self): + def convert(bw): + for suffix, factor in [("g", 1000**3), ("m", 1000**2), ("k", 1000)]: + if bw > 10 * factor: + return f'{bw/factor:.1f}{suffix}bps' + return f'{convert(self.down)}/{convert(self.up)}' + + def __eq__(self, o): + return (isinstance(o, Bandwidth) and + o.down == self.down and + o.up == self.up) + + +class LinkInfo: + def __init__(self, bandwidth, latency, jitter = None, offset = None, **kwargs): + self.bandwidth = Bandwidth(bandwidth) + self.latency = Latency(latency, offset) + self.jitter = Latency(jitter or 0) + + def __eq__(self, o): + return (isinstance(o, LinkInfo) and + o.bandwidth == self.bandwidth and + o.latency == self.latency and + o.jitter == self.jitter) + + def __str__(self): + return f'LinkInfo{{bw: {self.bandwidth}, latency: {self.latency}, jitter: {self.jitter}}}' + +class Server: + def __init__(self, name, link): + self.ip = None + self.name = name + self.link = link + + def is_zone(self): + return False + + def __str__(self): + return f'Server{{name: {self.name}, ip: {self.ip}, link: {self.link}}}' + + def __repr__(self): + return self.__str__() + +class Zone: + def __init__(self, name): + self.name = name + self.link = None + self.servers = {} + + def is_zone(self): + return True + + def add_server(self, server): + if self.servers.get(server.name) is None: + self.servers[server.name] = server + else: + raise Exception(f"Duplicate server '{server.name}' in zone '{self.name}'") + + def set_link(self, link): + if self.link is None: + self.link = link + elif self.link != link: + raise Exception(f"Uncoherent link configuration for zone '{self.name}'") + + def __str__(self): + return f'Zone{{name: {self.name}, link: {self.link}, servers: {list(self.servers.values())}}}' + + def __repr__(self): + return self.__str__() + + +class Network: + def __init__(self): + self.zones = {} + self.subnet_manager = None + self.latency_off = Latency(0) + self.host_ip = None + self.host_link = None + + def set_subnet_manager(self, subnet): + self.subnet_manager = SubnetManager(subnet) + + def set_latency_offset(self, latency_off): + self.latency_off = latency_off + + def add_server(self, server): + name = server["name"] + if zone_obj := server.get("zone"): + zone_name = zone_obj["name"] + if zone := self.zones.get(zone_name): + if not zone.is_zone(): + raise Exception("Duplicate zone: " + name) + else: + zone = Zone(zone_name) + self.zones[zone_name] = zone + if link:=zone_obj.get("external"): + zone.set_link(LinkInfo(offset = self.latency_off, **link)) + zone.add_server(Server(name, LinkInfo(**zone_obj["internal"]))) + + else: + name = name + if name in self.zones: + raise Exception("Duplicate zone: " + name) + self.zones[name] = Server(name, LinkInfo(offset = self.latency_off, **server)) + + def assign_ips(self): + for zone in self.zones.values(): + if zone.is_zone(): + for server in zone.servers.values(): + server.ip = self.subnet_manager.next_local() + else: + zone.ip = self.subnet_manager.next_local() + self.subnet_manager.next_zone() + if not self.host_ip: + self.host_ip = self.subnet_manager.next_local() + + def __str__(self): + return f'Network{{subnet_manager: {self.subnet_manager}, zones: {list(self.zones.values())}, latency_offset: {self.latency_off}}}' + +class NamespaceManager: + def __init__(self): + self.namespaces = set(["unconfined"]) + self.prefixlen = 0 + net.ns.name_unconfined() + + def make_namespace(self, name): + if not name in self.namespaces: + net.ns.create(name) + self.namespaces.add(name) + + def make_bridge(self, name, namespace, ports): + self.make_namespace(namespace) + net.create_bridge(name, namespace, ports) + + def make_veth(self, name1, name2, space1, space2, ip, link=None): + self.make_namespace(space1) + self.make_namespace(space2) + net.create_veth(name1, space1, name2, space2, ip, self.prefixlen, link) + + def build_network(self, network): + self.prefixlen = network.subnet_manager.prefix + netns = "testnet-core" + self.make_veth("veth-testnet", "unconfined", "unconfined", netns, network.host_ip, network.host_link) + ports = ["unconfined"] + for zone in network.zones.values(): + if zone.is_zone(): + self.build_zone(zone) + else: + self.build_server(zone) + ports.append('veth-' + zone.name) + self.make_bridge("br0", netns, ports) + + def build_zone(self, zone): + netns = "testnet-" + zone.name + self.make_veth("veth-" + zone.name, "veth-" + zone.name, netns, "testnet-core", None, zone.link) + ports = ['veth-' + zone.name] + for server in zone.servers.values(): + self.build_server(server, zone) + ports.append('veth-' + server.name) + self.make_bridge("br-" + zone.name, netns, ports) + + def build_server(self, server, zone = None): + if zone: + zone_name = "testnet-" + zone.name + namespace = zone_name + "-" + server.name + else: + zone_name = "testnet-core" + namespace = "testnet-" + server.name + "-" + server.name + self.make_veth("veth", "veth-" + server.name, namespace, zone_name, server.ip, server.link) + +def parse(yaml): + server_list = yaml["servers"] + global_conf = yaml.get("global", {}) + subnet = global_conf.get("subnet", {'base': 'fc00:9a7a:9e::', 'local': 64, 'zone': 16}) + latency_offset = global_conf.get("latency-offset", 0) + + network = Network() + if upstream := global_conf.get("upstream"): + network.host_ip = upstream.get("ip") + if host_link:= upstream.get("conn"): + network.host_link = LinkInfo(latency_offset=latency_offset, **host_link) + network.set_subnet_manager(subnet) + network.set_latency_offset(latency_offset) + for server in server_list: + network.add_server(server) + network.assign_ips() + return network + +def create(config_path): + with open(config_path, "r") as file: + config = yaml.safe_load(file) + shutil.copy(config_path, ".current_state.yml") + network = parse(config) + nsm = NamespaceManager() + nsm.build_network(network) + +def run(netns, cmd): + if ":" in netns: + zone_name,host = netns.split(":", 1) + else: + zone_name,host = None, netns + with open(".current_state.yml", "r") as file: + config = yaml.safe_load(file) + zones = parse(config).zones + server = None + zone = None + if zone_name: + if (zone := zones.get(zone_name)) and zone.is_zone(): + server = zone.servers.get(host) + elif (zone := zones.get(host)) and not zone.is_zone(): + server = zone + else: + for z in zones.values(): + if not z.is_zone(): continue + if (s := z.servers.get(host)): + if server: + raise Exception("Multiple matching host found.") + server = s + zone = z + + if not server: + raise Exception("No matching host was found") + + env = os.environ.copy() + env["HOST"] = server.name + if zone.is_zone(): + env["ZONE"] = zone.name + env["IP"] = str(server.ip) + name = f'testnet-{zone.name}-{server.name}' + + if len(cmd) == 0: + cmd = [os.getenv("SHELL") or "/bin/sh"] + os.execve("/usr/bin/env", ["/usr/bin/env", "ip", "netns" , "exec", name ] + cmd, env) + +def runall(cmd): + with open(".current_state.yml", "r") as file: + config = yaml.safe_load(file) + zones = parse(config).zones + + number = 1 + for zone in zones.values(): + if zone.is_zone(): + for server in zone.servers.values(): + env = os.environ.copy() + env["ZONE"] = zone.name + env["HOST"] = server.name + env["IP"] = str(server.ip) + env["ID"] = str(number) + env["SIZE"] = str(len(config['servers'])) + name = f'testnet-{zone.name}-{server.name}' + net.ns.run(name, cmd, env) + number +=1 + else: + env = os.environ.copy() + env["ZONE"] = "" + env["HOST"] = zone.name + env["IP"] = str(zone.ip) + env["ID"] = str(number) + env["SIZE"] = str(len(config['servers'])) + name = f'testnet-{zone.name}-{zone.name}' + net.ns.run(name, cmd, env) + first = False + number +=1 + +def destroy(): + for ns in net.ns.list(): + net.ns.kill(ns) + net.ns.forget("unconfined") + os.remove(".current_state.yml") + +if __name__ == "__main__": + if len(sys.argv) < 2: + progname = os.path.basename(sys.argv[0]) if len(sys.argv) > 0 else "mknet" + print(f"""Usage: + {progname} create [config_path] # create a new network. config_path defailt to config.yml + {progname} run-all [args...] # run a command as each host. set the IP, NAME and ZONE environment variables + {progname} run [cmd [args...]] # run command in host named . Use zonename:name if multiple zones hosts server with same name. If cmd is empty, run a shell + {progname} destroy # destroy the current environment""") + exit() + cmd = sys.argv[1] + if cmd == "create": + create(sys.argv[2] if len(sys.argv) > 2 else "config.yml") + elif cmd == "run": + run(sys.argv[2], sys.argv[3:]) + elif cmd == "run-all": + runall(sys.argv[2:]) + elif cmd == "destroy": + destroy() + else: + raise Exception(f"Unknown command: {cmd}") diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..fda29c5 --- /dev/null +++ b/setup.py @@ -0,0 +1,6 @@ +from distutils.core import setup +setup(name='mknet', + version='1.0', + scripts=['mknet'], + py_modules=['net'], + ) -- cgit v1.2.3 From 322e949de0b416118365fbfb9448e5e5a1be6b23 Mon Sep 17 00:00:00 2001 From: Quentin Date: Wed, 8 Dec 2021 16:30:28 +0100 Subject: Update documentation to use pip --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 270d616..43574e7 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,19 @@ # mknet -Show usage: +## Installation -```help -python3 main.py +```bash +sudo pip3 install https://git.deuxfleurs.fr/trinity-1686a/mknet ``` -## Example +## Usage ```bash sudo rm -rf /tmp/garage-testnet/ # always start by deleting previous run -sudo python3 main.py create ./config.yml -sudo python3 main.py run-all example/deploy_garage.sh -sudo python3 main.py run dc1:dc1s1 garage -c /tmp/garage-testnet/dc1/dc1s1/garage.toml status -sudo python3 main.py destroy +sudo mknet create ./config.yml +sudo mknet run-all example/deploy_garage.sh +sudo mknet run dc1:dc1s1 garage -c /tmp/garage-testnet/dc1/dc1s1/garage.toml status +sudo mknet destroy ``` ## Instrumented daemons -- cgit v1.2.3 From 69871eeee82c5e95e28fa94555a774d415cfdd90 Mon Sep 17 00:00:00 2001 From: Quentin Date: Wed, 8 Dec 2021 16:36:33 +0100 Subject: Remove MINIO_PATH --- example/deploy_minio.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/example/deploy_minio.py b/example/deploy_minio.py index 8ee8430..4488974 100755 --- a/example/deploy_minio.py +++ b/example/deploy_minio.py @@ -2,7 +2,6 @@ import json, os, sys, time, pathlib, socket, shutil STORAGE_PATH = os.path.join(os.getcwd(), '.minio-testnet') -MINIO_PATH = '/srv' HOSTS_PATH = os.path.join(STORAGE_PATH, 'hosts.txt') UNIX_SOCK = os.path.join(STORAGE_PATH, 'deploy.sock') DATA_PATH = lambda nid: os.path.join(STORAGE_PATH, 'data'+str(nid)) @@ -14,7 +13,6 @@ def main(): def leader(): shutil.rmtree(STORAGE_PATH, ignore_errors=True) os.makedirs(STORAGE_PATH) - os.makedirs(MINIO_PATH, exist_ok=True) print(STORAGE_PATH) sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) -- cgit v1.2.3 From 1607ceaf5c6d2e5f9b8656dedd53049c66eaa079 Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Wed, 8 Dec 2021 18:06:53 +0100 Subject: fix pip instruction --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 43574e7..92abc61 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ ## Installation ```bash -sudo pip3 install https://git.deuxfleurs.fr/trinity-1686a/mknet +sudo pip3 install git+https://git.deuxfleurs.fr/trinity-1686a/mknet ``` ## Usage -- cgit v1.2.3 From 9719f4b294b4d295ed62848c3a82a5271eb86139 Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Mon, 13 Dec 2021 10:53:42 +0100 Subject: Change env name + fix python casting --- example/deploy_minio.py | 4 ++-- mknet | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/example/deploy_minio.py b/example/deploy_minio.py index 4488974..431b983 100755 --- a/example/deploy_minio.py +++ b/example/deploy_minio.py @@ -19,7 +19,7 @@ def leader(): sock.bind(UNIX_SOCK) sock.listen() - n_serv = os.environ['SIZE'] + n_serv = int(os.environ['SERVER_COUNT']) fl = [ co for co, addr in [ sock.accept() for i in range(n_serv - 1) ]] identities = [ json.loads(co.makefile().readline()) for co in fl ] + [ { "ip": os.environ['IP'], "path": make_data() } ] @@ -52,7 +52,7 @@ def make_data(): return data_path def run_minio(identities): - cmd = f"minio server --console-address ':9001' --address [{os.environ['IP']}]:9000" + cmd = f"minio server --console-address ':9001' --address ':9000'" for ident in identities: cmd += f" http://[{ident['ip']}]:9000{ident['path']}" cmd += f" > {os.path.join(STORAGE_PATH, 'minio'+os.environ['ID']+'.log')} 2>&1" diff --git a/mknet b/mknet index 8eb276a..9081aeb 100755 --- a/mknet +++ b/mknet @@ -321,7 +321,7 @@ def runall(cmd): env["HOST"] = server.name env["IP"] = str(server.ip) env["ID"] = str(number) - env["SIZE"] = str(len(config['servers'])) + env["SERVER_COUNT"] = str(len(config['servers'])) name = f'testnet-{zone.name}-{server.name}' net.ns.run(name, cmd, env) number +=1 @@ -331,7 +331,7 @@ def runall(cmd): env["HOST"] = zone.name env["IP"] = str(zone.ip) env["ID"] = str(number) - env["SIZE"] = str(len(config['servers'])) + env["SERVER_COUNT"] = str(len(config['servers'])) name = f'testnet-{zone.name}-{zone.name}' net.ns.run(name, cmd, env) first = False -- cgit v1.2.3