diff options
| -rw-r--r-- | config.yml | 9 | ||||
| -rwxr-xr-x | example/deploy_garage.sh | 51 | ||||
| -rwxr-xr-x | main.py | 25 | ||||
| -rw-r--r-- | net.py | 14 |
4 files changed, 89 insertions, 10 deletions
@@ -40,7 +40,10 @@ servers: global: subnet: - base: 172.24.0.0 - local: 8 - zone: 8 + base: fc00:9a7a:9e:: + local: 64 + zone: 16 latency-offset: 3ms + upstream: + ip: fc00:9a7a:9e:ffff:ffff:ffff:ffff:ffff + conn: *fiber diff --git a/example/deploy_garage.sh b/example/deploy_garage.sh new file mode 100755 index 0000000..8377673 --- /dev/null +++ b/example/deploy_garage.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +set -euo pipefail +IFS=$'\n\t' + +GARAGE_PATH=/home/trinity/tmp/garage/target/release/garage +STORAGE_PATH=/tmp/garage-testnet +export RPC_SECRET=3e9abff5f9e480afbadb46a77b7a26fe0e404258f0dc3fd5386b0ba8e0ad2fba + +NODE_STORAGE_PATH=${STORAGE_PATH}/${ZONE}/${HOST} +BOOTSTRAP_FILE=${STORAGE_PATH}/bootstrap_peer +export GARAGE_CONFIG_FILE=${NODE_STORAGE_PATH}/garage.toml + + +mkdir -p ${NODE_STORAGE_PATH} +cd ${NODE_STORAGE_PATH} +rm ${BOOTSTRAP_FILE} 2>/dev/null || true + +cat > ${GARAGE_CONFIG_FILE} << EOF +metadata_dir = "${NODE_STORAGE_PATH}/meta" +data_dir = "${NODE_STORAGE_PATH}/data" + +replication_mode = "3" + +rpc_bind_addr = "[::]:3901" +rpc_public_addr = "[${IP}]:3901" +rpc_secret = "${RPC_SECRET}" + +bootstrap_peers=[] + +[s3_api] +s3_region = "garage" +api_bind_addr = "[::]:3900" + +[s3_web] +bind_addr = "[::]:3902" +root_domain = ".web.garage" +index = "index.html" +EOF + +RUST_LOG=garage=debug ${GARAGE_PATH} server 2>> ${NODE_STORAGE_PATH}/logs & disown +sleep 2 + +CONFIG_NODE_FPATH=(${STORAGE_PATH}/*{,/*}/garage.toml) + +SELF_ID=$(${GARAGE_PATH} node-id 2>/dev/null) +SHORT_ID=$(echo ${SELF_ID} | cut -c-64) + +${GARAGE_PATH} -c ${CONFIG_NODE_FPATH[0]} node connect ${SELF_ID} +sleep ${ID} +${GARAGE_PATH} -c ${CONFIG_NODE_FPATH[0]} node configure -z ${ZONE:-unzonned-${HOST}} -c 1 -t ${HOST} ${SHORT_ID} @@ -149,6 +149,8 @@ class Network: self.zones = {} self.subnet_manager = None self.latency_off = Latency(0) + self.host_ip = None + self.host_link = None def set_subnet_manager(self, subnet): self.subnet_manager = SubnetManager(subnet) @@ -184,14 +186,17 @@ class Network: else: zone.ip = self.subnet_manager.next_local() self.subnet_manager.next_zone() + if not self.host_ip: + self.host_ip = self.subnet_manager.next_local() def __str__(self): return f'Network{{subnet_manager: {self.subnet_manager}, zones: {list(self.zones.values())}, latency_offset: {self.latency_off}}}' class NamespaceManager: def __init__(self): - self.namespaces = set() + self.namespaces = set(["unconfined"]) self.prefixlen = 0 + net.ns.name_unconfined() def make_namespace(self, name): if not name in self.namespaces: @@ -202,7 +207,7 @@ class NamespaceManager: self.make_namespace(namespace) net.create_bridge(name, namespace, ports) - def make_veth(self, name1, name2, space1, space2, ip, link): + def make_veth(self, name1, name2, space1, space2, ip, link=None): self.make_namespace(space1) self.make_namespace(space2) net.create_veth(name1, space1, name2, space2, ip, self.prefixlen, link) @@ -210,7 +215,8 @@ class NamespaceManager: def build_network(self, network): self.prefixlen = network.subnet_manager.prefix netns = "testnet-core" - ports = [] + self.make_veth("veth-testnet", "unconfined", "unconfined", netns, network.host_ip, network.host_link) + ports = ["unconfined"] for zone in network.zones.values(): if zone.is_zone(): self.build_zone(zone) @@ -244,6 +250,10 @@ def parse(yaml): latency_offset = global_conf.get("latency-offset", 0) network = Network() + if upstream := global_conf.get("upstream"): + network.host_ip = upstream.get("ip") + if host_link:= upstream.get("conn"): + network.host_link = LinkInfo(latency_offset=latency_offset, **host_link) network.set_subnet_manager(subnet) network.set_latency_offset(latency_offset) for server in server_list: @@ -302,6 +312,7 @@ def runall(cmd): config = yaml.safe_load(file) zones = parse(config).zones + number = 1 for zone in zones.values(): if zone.is_zone(): for server in zone.servers.values(): @@ -309,19 +320,25 @@ def runall(cmd): env["ZONE"] = zone.name env["HOST"] = server.name env["IP"] = str(server.ip) + env["ID"] = str(number) name = f'testnet-{zone.name}-{server.name}' net.ns.run(name, cmd, env) + number +=1 else: env = os.environ.copy() + env["ZONE"] = "" env["HOST"] = zone.name env["IP"] = str(zone.ip) + env["ID"] = str(number) name = f'testnet-{zone.name}-{zone.name}' net.ns.run(name, cmd, env) - + first = False + number +=1 def destroy(): for ns in net.ns.list(): net.ns.kill(ns) + net.ns.forget("unconfined") os.remove(".current_state.yml") if __name__ == "__main__": @@ -1,4 +1,5 @@ import subprocess +import os _netns = ["ip", "netns"] def run_netns(*cmd): @@ -8,9 +9,16 @@ def run_netns(*cmd): return process class ns: - def list(): - process = run_netns("list") - return [ns for ns in process.stdout.split("\n") if ns.startswith("testnet-")] + def name_unconfined(): + if "unconfined" not in ns.list(True): + run_netns("attach", "unconfined", str(os.getpid())) + + def list(include_unconfined = False): + try: + nss = os.listdir("/var/run/netns") + return [ns for ns in nss if ns.startswith("testnet-") or include_unconfined and ns == "unconfined"] + except FileNotFoundError: + return [] def forget(name): run_netns("del", name) |