add support for upstream port and add garage example script

4 files changed, 89 insertions, 10 deletions

diff --git a/config.yml b/config.yml
index 4b1cea3..327a3b1 100644
--- a/config.yml
+++ b/config.yml
@@ -40,7 +40,10 @@ servers:
 
 global:
   subnet:
-    base: 172.24.0.0
-    local: 8
-    zone: 8
+    base: fc00:9a7a:9e::
+    local: 64
+    zone: 16
   latency-offset: 3ms
+  upstream:
+    ip: fc00:9a7a:9e:ffff:ffff:ffff:ffff:ffff
+    conn: *fiber
diff --git a/example/deploy_garage.sh b/example/deploy_garage.sh
new file mode 100755
index 0000000..8377673
--- /dev/null
+++ b/example/deploy_garage.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+set -euo pipefail
+IFS=$'\n\t'
+
+GARAGE_PATH=/home/trinity/tmp/garage/target/release/garage
+STORAGE_PATH=/tmp/garage-testnet
+export RPC_SECRET=3e9abff5f9e480afbadb46a77b7a26fe0e404258f0dc3fd5386b0ba8e0ad2fba
+
+NODE_STORAGE_PATH=${STORAGE_PATH}/${ZONE}/${HOST}
+BOOTSTRAP_FILE=${STORAGE_PATH}/bootstrap_peer
+export GARAGE_CONFIG_FILE=${NODE_STORAGE_PATH}/garage.toml 
+
+
+mkdir -p ${NODE_STORAGE_PATH}
+cd ${NODE_STORAGE_PATH}
+rm ${BOOTSTRAP_FILE} 2>/dev/null || true
+
+cat > ${GARAGE_CONFIG_FILE} << EOF
+metadata_dir = "${NODE_STORAGE_PATH}/meta"
+data_dir = "${NODE_STORAGE_PATH}/data"
+
+replication_mode = "3"
+
+rpc_bind_addr = "[::]:3901"
+rpc_public_addr = "[${IP}]:3901"
+rpc_secret = "${RPC_SECRET}"
+
+bootstrap_peers=[]
+
+[s3_api]
+s3_region = "garage"
+api_bind_addr = "[::]:3900"
+
+[s3_web]
+bind_addr = "[::]:3902"
+root_domain = ".web.garage"
+index = "index.html"
+EOF
+
+RUST_LOG=garage=debug ${GARAGE_PATH} server 2>> ${NODE_STORAGE_PATH}/logs & disown
+sleep 2
+
+CONFIG_NODE_FPATH=(${STORAGE_PATH}/*{,/*}/garage.toml)
+
+SELF_ID=$(${GARAGE_PATH} node-id 2>/dev/null)
+SHORT_ID=$(echo ${SELF_ID} | cut -c-64)
+
+${GARAGE_PATH} -c ${CONFIG_NODE_FPATH[0]} node connect ${SELF_ID}
+sleep ${ID}
+${GARAGE_PATH} -c ${CONFIG_NODE_FPATH[0]} node configure -z ${ZONE:-unzonned-${HOST}} -c 1 -t ${HOST} ${SHORT_ID}
diff --git a/main.py b/main.py
index 8e236d6..c243a2f 100755
--- a/main.py
+++ b/main.py
@@ -149,6 +149,8 @@ class Network:
         self.zones = {}
         self.subnet_manager = None
         self.latency_off = Latency(0)
+        self.host_ip = None
+        self.host_link = None
 
     def set_subnet_manager(self, subnet):
         self.subnet_manager = SubnetManager(subnet)
@@ -184,14 +186,17 @@ class Network:
             else:
                 zone.ip = self.subnet_manager.next_local()
             self.subnet_manager.next_zone()
+        if not self.host_ip:
+            self.host_ip = self.subnet_manager.next_local()
 
     def __str__(self):
         return f'Network{{subnet_manager: {self.subnet_manager}, zones: {list(self.zones.values())}, latency_offset: {self.latency_off}}}'
 
 class NamespaceManager:
     def __init__(self):
-        self.namespaces = set()
+        self.namespaces = set(["unconfined"])
         self.prefixlen = 0
+        net.ns.name_unconfined()
 
     def make_namespace(self, name):
         if not name in self.namespaces:
@@ -202,7 +207,7 @@ class NamespaceManager:
         self.make_namespace(namespace)
         net.create_bridge(name, namespace, ports)
         
-    def make_veth(self, name1, name2, space1, space2, ip, link):
+    def make_veth(self, name1, name2, space1, space2, ip, link=None):
         self.make_namespace(space1)
         self.make_namespace(space2)
         net.create_veth(name1, space1, name2, space2, ip, self.prefixlen, link)
@@ -210,7 +215,8 @@ class NamespaceManager:
     def build_network(self, network):
         self.prefixlen = network.subnet_manager.prefix
         netns = "testnet-core"
-        ports = []
+        self.make_veth("veth-testnet", "unconfined", "unconfined", netns, network.host_ip, network.host_link)
+        ports = ["unconfined"]
         for zone in network.zones.values():
             if zone.is_zone():
                 self.build_zone(zone)
@@ -244,6 +250,10 @@ def parse(yaml):
     latency_offset = global_conf.get("latency-offset", 0)
 
     network = Network()
+    if upstream := global_conf.get("upstream"):
+        network.host_ip = upstream.get("ip")
+        if host_link:= upstream.get("conn"):
+            network.host_link = LinkInfo(latency_offset=latency_offset, **host_link)
     network.set_subnet_manager(subnet)
     network.set_latency_offset(latency_offset)
     for server in server_list:
@@ -302,6 +312,7 @@ def runall(cmd):
         config = yaml.safe_load(file)
     zones = parse(config).zones
 
+    number = 1
     for zone in zones.values():
         if zone.is_zone():
             for server in zone.servers.values():
@@ -309,19 +320,25 @@ def runall(cmd):
                 env["ZONE"] = zone.name
                 env["HOST"] = server.name
                 env["IP"] = str(server.ip)
+                env["ID"] = str(number)
                 name = f'testnet-{zone.name}-{server.name}'
                 net.ns.run(name, cmd, env)
+                number +=1
         else:
             env = os.environ.copy()
+            env["ZONE"] = ""
             env["HOST"] = zone.name
             env["IP"] = str(zone.ip)
+            env["ID"] = str(number)
             name = f'testnet-{zone.name}-{zone.name}'
             net.ns.run(name, cmd, env)
-
+            first = False
+            number +=1
 
 def destroy():
     for ns in net.ns.list():
         net.ns.kill(ns)
+    net.ns.forget("unconfined")
     os.remove(".current_state.yml")
 
 if __name__ == "__main__":
diff --git a/net.py b/net.py
index bc20e4d..8d0a248 100644
--- a/net.py
+++ b/net.py
@@ -1,4 +1,5 @@
 import subprocess
+import os
 
 _netns = ["ip", "netns"]
 def run_netns(*cmd):
@@ -8,9 +9,16 @@ def run_netns(*cmd):
     return process
 
 class ns:
-    def list():
-        process = run_netns("list")
-        return [ns for ns in process.stdout.split("\n") if ns.startswith("testnet-")]
+    def name_unconfined():
+        if "unconfined" not in ns.list(True):
+            run_netns("attach", "unconfined", str(os.getpid()))
+
+    def list(include_unconfined = False):
+        try:
+            nss = os.listdir("/var/run/netns")
+            return [ns for ns in nss if ns.startswith("testnet-") or include_unconfined and ns == "unconfined"]
+        except FileNotFoundError:
+            return []
 
     def forget(name):
         run_netns("del", name)