diff options
author | Alex Auvolat <alex@adnab.me> | 2018-11-06 15:03:22 +0100 |
---|---|---|
committer | Alex Auvolat <alex@adnab.me> | 2018-11-06 15:03:22 +0100 |
commit | 2973cf99c5b677c71717d916f83212bc2e6b36dc (patch) | |
tree | 8651b326786c62fc4e0bc438a625a13bf5df1844 /shard/lib | |
parent | c4f6cbab20b0b1d08755073d93365e5bd00dc755 (diff) | |
download | shard-2973cf99c5b677c71717d916f83212bc2e6b36dc.tar.gz shard-2973cf99c5b677c71717d916f83212bc2e6b36dc.zip |
Document
Diffstat (limited to 'shard/lib')
-rw-r--r-- | shard/lib/app/chat.ex | 47 | ||||
-rw-r--r-- | shard/lib/app/directory.ex | 21 | ||||
-rw-r--r-- | shard/lib/app/file.ex | 23 | ||||
-rw-r--r-- | shard/lib/app/identity.ex | 8 | ||||
-rw-r--r-- | shard/lib/app/pagestore.ex | 52 | ||||
-rw-r--r-- | shard/lib/cli/cli.ex | 21 | ||||
-rw-r--r-- | shard/lib/data/data.ex | 22 | ||||
-rw-r--r-- | shard/lib/data/merklelist.ex | 8 | ||||
-rw-r--r-- | shard/lib/data/merklesearchtree.ex | 16 | ||||
-rw-r--r-- | shard/lib/data/merkletree.ex | 14 | ||||
-rw-r--r-- | shard/lib/data/signrev.ex | 2 | ||||
-rw-r--r-- | shard/lib/data/store.ex | 11 | ||||
-rw-r--r-- | shard/lib/keys.ex | 30 | ||||
-rw-r--r-- | shard/lib/manager.ex | 44 | ||||
-rw-r--r-- | shard/lib/net/addr.ex | 18 | ||||
-rw-r--r-- | shard/lib/net/auth.ex | 10 | ||||
-rw-r--r-- | shard/lib/net/group.ex | 12 | ||||
-rw-r--r-- | shard/lib/net/manager.ex | 12 | ||||
-rw-r--r-- | shard/lib/net/tcpconn.ex | 15 | ||||
-rw-r--r-- | shard/lib/net/tcpserver.ex | 4 | ||||
-rw-r--r-- | shard/lib/shard_uri.ex | 9 |
21 files changed, 290 insertions, 109 deletions
diff --git a/shard/lib/app/chat.ex b/shard/lib/app/chat.ex index ff0c97d..b046fc3 100644 --- a/shard/lib/app/chat.ex +++ b/shard/lib/app/chat.ex @@ -12,11 +12,7 @@ defmodule SApp.Chat do %SApp.Chat.PrivChat.Manifest{pk_list: ordered_list_of_authorized_pks} Future improvements: - - message signing - - storage of the chatroom messages to disk - use a DHT to find peers that are interested in this channel - - epidemic broadcast (carefull not to be too costly, - maybe by limiting the number of peers we talk to) - partial synchronization only == data distributed over peers """ @@ -32,7 +28,9 @@ defmodule SApp.Chat do defmodule Manifest do @moduledoc""" - Manifest for a public chat room defined by its name. + Manifest for a public chat room defined by its name. Example: + + %SApp.Chat.Manifest{channel: "test"} """ defstruct [:channel] @@ -73,19 +71,22 @@ defmodule SApp.Chat do # ========== defmodule State do + @moduledoc""" + Internal state struct of chat shard. + """ + defstruct [:id, :netgroup, :manifest, :page_store, :mst, :subs, :read] end @doc """ - Start a process that connects to a given channel + Start a process that connects to a given channel. Don't call directly, use for instance: + + Shard.Manager.find_or_start %SApp.Chat.Manifest{channel: "my_chan"} """ def start_link(manifest) do GenServer.start_link(__MODULE__, manifest) end - @doc """ - Initialize channel process. - """ def init(manifest) do id = SData.term_hash manifest @@ -103,7 +104,7 @@ defmodule SApp.Chat do end root = cond do root == nil -> nil - GenServer.call(page_store, {:have_rec, root}) -> root + SApp.PageStore.have_rec?(page_store, root) -> root true -> Logger.warn "Not all pages for saved root were saved, restarting from an empty state!" nil @@ -124,9 +125,6 @@ defmodule SApp.Chat do } end - @doc """ - Implementation of the :manifest call that returns the chat room's manifest - """ def handle_call(:manifest, _from, state) do {:reply, state.manifest, state} end @@ -166,11 +164,6 @@ defmodule SApp.Chat do {:noreply, state} end - @doc """ - Implementation of the :chat_send handler. This is the main handler that is used - to send a message to the chat room. Puts the message in the store and syncs - with all connected peers. - """ def handle_cast({:chat_send, pk, msg}, state) do next_ts = case MST.last(state.mst, nil, 1) do [] -> System.os_time :seconds @@ -204,10 +197,6 @@ defmodule SApp.Chat do {:noreply, state} end - @doc """ - Implementation of the :interested handler, this is called when a peer we are - connected to asks to recieve data for this channel. - """ def handle_cast({:interested, conn_pid, auth}, state) do if SNet.Group.in_group?(state.netgroup, conn_pid, auth) do SNet.Manager.send_pid(conn_pid, {state.id, nil, {:root, state.mst.root, true}}) @@ -221,10 +210,6 @@ defmodule SApp.Chat do {:noreply, %{ state | subs: new_subs }} end - @doc """ - Implementation of the :msg handler, which is the main handler for messages - comming from other peers concerning this chat room. - """ def handle_cast({:msg, conn_pid, auth, _shard_id, nil, msg}, state) do if not SNet.Group.in_group?(state.netgroup, conn_pid, auth) do # Ignore message @@ -245,7 +230,7 @@ defmodule SApp.Chat do mst2 = MST.insert(state.mst, msgitem) if mst2.root == new_root do state = %{state | mst: mst2} - GenServer.cast(state.page_store, {:set_roots, [mst2.root]}) + SApp.PageStore.set_roots(state.page_store, [mst2.root]) save_state(state) msg_callback(state, msgitem) SNet.Group.broadcast(state.netgroup, {state.id, nil, msg}, exclude_pid: [conn_pid]) @@ -311,7 +296,7 @@ defmodule SApp.Chat do for x <- new do msg_callback(state, x) end - GenServer.cast(state.page_store, {:set_roots, [mst.root]}) + SApp.PageStore.set_roots(state.page_store, [mst.root]) state = %{state | mst: mst} save_state(state) if state.mst.root != old_root do @@ -365,15 +350,15 @@ defmodule SApp.Chat do The process calling this function will start recieving messages of the form: - {:chat_recv, manifest, {pk, msgbin, sign}} + {:chat_recv, manifest, {pk, msgbin, sign}} or - {:chat_send, manifest, {pk, msgbin, sign}} + {:chat_send, manifest, {pk, msgbin, sign}} msgbin can be used in the following way: - {timestamp, message} = SData.term_unbin msgbin + {timestamp, message} = SData.term_unbin msgbin """ def subscribe(shard_pid) do GenServer.cast(shard_pid, {:subscribe, self()}) diff --git a/shard/lib/app/directory.ex b/shard/lib/app/directory.ex index cbea8c3..257e8b9 100644 --- a/shard/lib/app/directory.ex +++ b/shard/lib/app/directory.ex @@ -29,9 +29,18 @@ defmodule SApp.Directory do end defmodule State do + @moduledoc""" + Internal state struct of directory shard. + """ + defstruct [:owner, :public, :name, :manifest, :id, :netgroup, :items, :revitems] end + @doc""" + Start a process that connects to a given channel. Don't call directly, use for instance: + + Shard.Manager.find_or_start %SApp.Directory.Manifest{owner: my_pk, public: false, name: "collection"} + """ def start_link(manifest) do GenServer.start_link(__MODULE__, manifest) end @@ -217,23 +226,23 @@ defmodule SApp.Directory do @doc""" Return list of items stored in this directory. - Returns a dictionnary of %{name => {manifest, stored?}}. + Returns a dictionnary of `%{name => {manifest, stored?}}`. """ def get_items(pid) do GenServer.call(pid, :get_items) end @doc""" - Return the manifest of item with a given name in directory, or nil if not found. + Return the manifest of item with a given name in directory, or `nil` if not found. - Equivalent to get_items(pid)[name] but better. + Equivalent to `get_items(pid)[name]` but better. """ def read(pid, name) do GenServer.call(pid, {:read, name}) end @doc""" - Find an item in the directory by its manifest. Returns name if found or nil if not found. + Find an item in the directory by its manifest. Returns name if found or `nil` if not found. """ def find(pid, manifest) do GenServer.call(pid, {:find, manifest}) @@ -241,8 +250,8 @@ defmodule SApp.Directory do @doc""" Add an item to this directory. An item is a name for a shard manifest. - An item added to a directory becomes a dependency of the directory, i.e. - if the directory is pinned then all items inside are pinned as well. + An item added to a directory with `stored = true` becomes a dependency of the directory, + i.e. if the directory is pinned then all items inside are pinned as well. """ def add_item(pid, name, manifest, stored \\ true) do GenServer.call(pid, {:add_item, name, manifest, stored}) diff --git a/shard/lib/app/file.ex b/shard/lib/app/file.ex index e2a9798..0e07cc3 100644 --- a/shard/lib/app/file.ex +++ b/shard/lib/app/file.ex @@ -9,9 +9,12 @@ defmodule SApp.File do file_hash: hash size: int mime_type: string - } + } + + The file is cut in blocks that are collected in a k-ary Merkle tree + (see SData.MerkleTree for block size and k value). - The file is cut in blocks of 4kb that are collected in a 64-ary Merkle tree. + TODO I feel bad about some parts of the logic in here. """ use GenServer @@ -26,7 +29,7 @@ defmodule SApp.File do defmodule Manifest do @moduledoc""" Manifest for a file. - The file is identified by the root hash of its Merkle tree and by its mime type. + The file is identified by its infohash, which is the hash of a `SApp.File.Info` struct. """ defstruct [:infohash] @@ -46,9 +49,21 @@ defmodule SApp.File do end defmodule State do + @moduledoc""" + Internal state struct for file shard. + """ defstruct [:infohash, :id, :manifest, :netgroup, :info, :infobin, :store, :missing, :path, :reqs] end + @doc """ + Start a process that connects to a given channel. Don't call directly, use for instance: + + Shard.Manager.find_or_start %SApp.File.Manifest{infohash: "some_infohash"} + + or: + + SApp.File.Create("/path/to/file", "mime/type") + """ def start_link(manifest) do GenServer.start_link(__MODULE__, manifest) end @@ -229,7 +244,7 @@ defmodule SApp.File do true -> meta = get_mt(state) n_blocks = MT.block_count(meta) - expected_hashes = MT.get_range(meta, 0..(n_blocks-1)) + expected_hashes = MT.get_all(meta) actual_hashes = if File.exists?(state.path) do File.stream!(state.path, [], MT.block_size()) |> Enum.map(&(:crypto.hash(:sha256, &1))) diff --git a/shard/lib/app/identity.ex b/shard/lib/app/identity.ex index 78abbe7..7422822 100644 --- a/shard/lib/app/identity.ex +++ b/shard/lib/app/identity.ex @@ -34,9 +34,17 @@ defmodule SApp.Identity do end defmodule State do + @moduledoc""" + Internal state struct for identity shard. + """ defstruct [:pk, :id, :state, :netgroup] end + @doc """ + Start a process that connects to a given channel. Don't call directly, use for instance: + + Shard.Manager.find_or_start %SApp.Identity.Manifest{pk: some_public_key} + """ def start_link(manifest) do GenServer.start_link(__MODULE__, manifest) end diff --git a/shard/lib/app/pagestore.ex b/shard/lib/app/pagestore.ex index 3cda51d..0cbb10a 100644 --- a/shard/lib/app/pagestore.ex +++ b/shard/lib/app/pagestore.ex @@ -7,12 +7,29 @@ defmodule SApp.PageStore do Uses an ETS table of: - { page_id, why_have_it } -- waiting for data - { page_id, why_have_it, data } -- once we have the data + { page_id, why_have_it } # waiting for data + { page_id, why_have_it, data } # once we have the data - why_have_it := :root - | {:req_by, some_other_page_id} - | {:cached, expiry_date} + why_have_it := :root + | {:req_by, some_other_page_id} + | {:cached, expiry_date} + + TODO: at the moment we are trying to pull all missing pages at once from our peers. + This can work for metadata that isn't too big but won't work with bigger objects. + Have a smart strategy where we limit the number of requests currently in-flight but + still make sure everything gets pulled in. This will also pave the way to selectively + pulling in pages, for instance if we have a function to give them a priority score and + a maximum stored page count. + + A `SApp.PageStore` can be used as a `SData.PageStore` in the following way: + + %SApp.PageStore{pid: store_pid} + + or: + + %SApp.PageStore{pid: store_pid, prefer_ask: [connection_pid, ...]} + + In the second case, missing pages will be requested first to the specified peers. """ use GenServer @@ -25,6 +42,9 @@ defmodule SApp.PageStore do @max_failures 4 # Maximum of peers that reply not_found before we abandon defmodule State do + @moduledoc""" + Internal state struct of pagestore process. + """ defstruct [:shard_id, :path, :netgroup, :store, :reqs, :retries, :store_path] end @@ -258,7 +278,7 @@ defmodule SApp.PageStore do {:noreply, state} end - def ask_random_peers(state, key) do + defp ask_random_peers(state, key) do SNet.Group.broadcast(state.netgroup, {state.shard_id, state.path, {:get, key}}, nmax: 3) end @@ -289,4 +309,24 @@ defmodule SApp.PageStore do store ## DO SOMETHING??? end end + + # ==================== + # PAGE STORE INTERFACE + # ==================== + + @doc""" + Returns `true` if the page store currently stores the specified root page + and all its dependencies, recursively. + """ + def have_rec?(pid, root) do + GenServer.call(pid, {:have_rec, root}) + end + + @doc""" + Define the set of root pages we are interested in. This will start pulling in + the defined pages and all their dependencies recursively if we don't have them. + """ + def set_roots(pid, roots) do + GenServer.cast(pid, {:set_roots, roots}) + end end diff --git a/shard/lib/cli/cli.ex b/shard/lib/cli/cli.ex index 54b882f..8495b93 100644 --- a/shard/lib/cli/cli.ex +++ b/shard/lib/cli/cli.ex @@ -1,12 +1,23 @@ defmodule SCLI do @moduledoc """ - Small command line interface for the chat application + Small command line interface for the chat application. Supports public chat rooms, + private conversations, sending files (but not receiving them - could be done easily). + + The code of this module is intended as an example of how to use the Shard library. + + TODO: more commands. """ defmodule State do + @moduledoc""" + Internal state struct of the CLI. + """ defstruct [:room_pid, :id_pid, :pk] end + @doc""" + Call this from the iex prompt to launch the CLI. + """ def run() do for {_chid, manifest, _} <- Shard.Manager.list_shards do case manifest do @@ -197,6 +208,14 @@ defmodule SCLI do state end + defp handle_command(state, ["shards"]) do + Shard.Manager.list_shards + |> Enum.map(&(ShardURI.from_manifest(elem(&1, 1)))) + |> Enum.sort() + |> Enum.map(&IO.puts/1) + state + end + defp handle_command(state, _cmd) do IO.puts "Invalid command" state diff --git a/shard/lib/data/data.ex b/shard/lib/data/data.ex index 33dca09..8d2b277 100644 --- a/shard/lib/data/data.ex +++ b/shard/lib/data/data.ex @@ -10,34 +10,46 @@ defmodule SData do These functions must only return :duplicate for equal items. """ - @doc """ + @doc""" Calculate the hash of an Erlang term by first converting it to its - binary representation. + binary representation. Equivalent to `bin_hash(term_bin(term))`. """ def term_hash(term, algo \\ :sha256) do :crypto.hash(algo, (:erlang.term_to_binary term)) end + @doc""" + Convert any Erlang term to a binary representation. + """ def term_bin(term) do :erlang.term_to_binary term end + @doc""" + Calculate the hash of a binary. + """ def bin_hash(bin, algo \\ :sha256) do :crypto.hash(algo, bin) end + @doc""" + Calculate the hash of a file. + """ def file_hash(path, algo \\ :sha256) do File.stream!(path, [], 65536) |> Enum.reduce(:crypto.hash_init(algo), &(:crypto.hash_update(&2, &1))) |> :crypto.hash_final() end + @doc""" + Recover an Erlang term from its binary representation. + """ def term_unbin(bin) do :erlang.binary_to_term(bin, [:safe]) end @doc""" - Compare function for arbitrary terms using the Erlang order + Compare function for arbitrary terms using the Erlang order """ def cmp_term(a, b) do cond do @@ -48,7 +60,7 @@ defmodule SData do end @doc""" - Compare function for timestamped strings + Compare function for timestamped strings """ def cmp_ts_str({ts1, str1}, {ts2, str2}) do cond do @@ -61,7 +73,7 @@ defmodule SData do end @doc""" - Merge function for nils + Merge function for nils """ def merge_true(true, true), do: true end diff --git a/shard/lib/data/merklelist.ex b/shard/lib/data/merklelist.ex index 9b44ee8..c450ca7 100644 --- a/shard/lib/data/merklelist.ex +++ b/shard/lib/data/merklelist.ex @@ -1,12 +1,8 @@ defmodule SData.MerkleList do @moduledoc""" - A simple Merkle list store. + A simple Merkle list store. Not used. - Future improvements: - - When messages are inserted other than at the top, all intermediate hashes - change. Keep track of the mapping from old hashes to new hashes so that get - requests can work even for hashes that are not valid anymore. - - group items in "pages" (bigger bundles) + TODO delete this module """ defstruct [:root, :top, :cmp, :store] diff --git a/shard/lib/data/merklesearchtree.ex b/shard/lib/data/merklesearchtree.ex index e646774..f67843d 100644 --- a/shard/lib/data/merklesearchtree.ex +++ b/shard/lib/data/merklesearchtree.ex @@ -3,15 +3,15 @@ defmodule SData.MerkleSearchTree do A Merkle search tree. A node of the tree is - { - level, - hash_of_node | nil, - [ - { item_low_bound, hash_of_node | nil }, - { item_low_bound, hash_of_node | nil }, - ... + { + level, + hash_of_node | nil, + [ + { item_low_bound, hash_of_node | nil }, + { item_low_bound, hash_of_node | nil }, + ... + } } - } """ alias SData.PageStore, as: Store diff --git a/shard/lib/data/merkletree.ex b/shard/lib/data/merkletree.ex index 73679cf..94bd443 100644 --- a/shard/lib/data/merkletree.ex +++ b/shard/lib/data/merkletree.ex @@ -92,9 +92,17 @@ defmodule SData.MerkleTree do end @doc""" - Get the hashes of all blocks in a range + Get the hashes of all blocks """ - def get_range(mt, range) do - range |> Enum.map(&(get(mt, &1))) # TODO: do this efficiently + def get_all(mt) do + %Page{child_nblk: cn, list: list} = Store.get(mt.store, mt.root) + if cn == 1 do + list + else + list + |> Enum.map(&(%{mt | root: &1})) + |> Enum.map(&get_all/1) + |> Enum.reduce([], &(&2++&1)) + end end end diff --git a/shard/lib/data/signrev.ex b/shard/lib/data/signrev.ex index 6360b53..164df03 100644 --- a/shard/lib/data/signrev.ex +++ b/shard/lib/data/signrev.ex @@ -56,7 +56,7 @@ defmodule SData.SignRev do @doc""" Check that a signed binary is correct and merge it into the SignRev. - Returns {true, new_sr} if an update happenned, {false, sr} otherwise. + Returns `{true, new_sr}` if an update happenned, `{false, sr}` otherwise. """ def merge(sr, signed, pk) do case Shard.Keys.open(pk, signed) do diff --git a/shard/lib/data/store.ex b/shard/lib/data/store.ex index ca12cd0..ce5618c 100644 --- a/shard/lib/data/store.ex +++ b/shard/lib/data/store.ex @@ -24,6 +24,10 @@ defprotocol SData.PageStore do This protocol may also be implemented by store proxies that track operations and implement different synchronization or caching mechanisms. + + A page store is an object that stores data pages (arbitrary Erlang terms) and + identifies them by their hash. Dependencies may exist between pages, in which + case they form a Merkle DAG. """ @doc""" @@ -60,8 +64,15 @@ end defmodule SData.LocalStore do + @moduledoc""" + A page store that saves all pages locally in RAM. The store is basically a dictionnary + of hash to term mappings, which is mutated by put operations. + """ defstruct [:pages] + @doc""" + Create empty LocalStore. + """ def new() do %SData.LocalStore{ pages: %{} } end diff --git a/shard/lib/keys.ex b/shard/lib/keys.ex index 412baa2..3a97b5f 100644 --- a/shard/lib/keys.ex +++ b/shard/lib/keys.ex @@ -1,6 +1,6 @@ defmodule Shard.Keys do @moduledoc""" - Module for saving private keys. + Module for saving private keys, signing messages and checking message signatures. """ use Agent @@ -39,6 +39,10 @@ defmodule Shard.Keys do :binary.longest_common_suffix([pk, suffix]) == byte_size(suffix) end + @doc""" + Return any public key for which we have the secret key. Generates a new keypair + if necessary. + """ def get_any_identity() do Agent.get(__MODULE__, fn _ -> case list_identities() do @@ -96,6 +100,9 @@ defmodule Shard.Keys do end end + @doc""" + Check if we have the secret key associated with a public key. + """ def have_sk?(pk) do case :dets.lookup @key_db, pk do [{^pk, _sk}] -> true @@ -103,6 +110,9 @@ defmodule Shard.Keys do end end + @doc""" + Return the secret key associated with a public key if we have it or `nil` otherwise. + """ def get_sk(pk) do case :dets.lookup @key_db, pk do [{^pk, sk}] -> sk @@ -111,12 +121,12 @@ defmodule Shard.Keys do end @doc""" - Lookup the secret key for a pk and generate a detached signature for a message. + Lookup the secret key for a pk and generate a detached signature for a message. - The original message is not returned. + The original message is not returned. - Answer is {:ok, signature} if it worked, or :not_found if we didn't find the key. - + Answer is {:ok, signature} if it worked, or :not_found if we don't have the corresponding + secret key. """ def sign_detached(pk, bin) do case :dets.lookup @key_db, pk do @@ -127,9 +137,9 @@ defmodule Shard.Keys do end @doc""" - Verify a detached signature for a message + Verify a detached signature for a message - Returns :ok if the signature was correct. + Returns :ok if the signature was correct. """ def verify(pk, bin, sign) do if valid_identity_pk? pk do @@ -143,12 +153,16 @@ defmodule Shard.Keys do end @doc""" - Check if a public key is a valid identity pk. Requirement: have the correct suffix. + Check if a public key is a valid identity pk. Requirement: have the correct suffix. """ def valid_identity_pk?(pk) do check_suffix(pk, Application.get_env(:shard, :identity_suffix)) end + @doc""" + Creates a displayable representation of a public key by taking the hex representation + of its first four bytes. (not tamper proof but better than nothing) + """ def pk_display(pk) do pk |> binary_part(0, 4) diff --git a/shard/lib/manager.ex b/shard/lib/manager.ex index c3897a3..ed21380 100644 --- a/shard/lib/manager.ex +++ b/shard/lib/manager.ex @@ -14,7 +14,8 @@ defprotocol Shard.Manifest do """ @doc""" - Get the module in question. + Get the module that implements the shard. All shard modules must have a function + `start_link` that start the shard process and take a single argument: the manifest. """ def module(manifest) @@ -26,29 +27,38 @@ end defmodule Shard.Manager do @moduledoc""" - Maintains several important tables : + The manager is the main process by which shards are started, stopped, and their lifetime + monitored. - - :shard_db (persistent with DETS) - - List of - { id, manifest, why_have_it, state } + Maintains several important tables : - why_have_it := {:pinned, %MapSet{who requires it...}, %MapSet{who it requires...}} - | {:req, %MapSet{who requires it...}, %MapSet{who it requires...}} - | {:cached, expiry_date} + - `@shard_db` (persistent with DETS), a list of: + + ``` + { id, manifest, why_have_it, state } + + why_have_it := {:pinned, %MapSet{who requires it...}, %MapSet{who it requires...}} + | {:req, %MapSet{who requires it...}, %MapSet{who it requires...}} + | {:cached, expiry_date} + ``` + + - `@peer_db` (persistent with DETS), a multi-list of: - - :peer_db (persistent with DETS) + ``` + { shard_id, peer_info } # TODO: add health info (last seen, ping, etc) - Mult-list of - { shard_id, peer_info } # TODO: add health info (last seen, ping, etc) + peer_info := {:inet, ip, port} + TODO peer_info |= {:inet6, ip, port} | {:onion, name} + ``` - peer_info := {:inet, ip, port} - TODO peer_info |= {:inet6, ip, port} | {:onion, name} + - `:shard_procs` (not persistent), a list of: - - :shard_procs (not persistent) + ``` + { {id, path}, pid } + ``` - List of - { {id, path}, pid } + The path value is used to distinguish between a shard's main process (`path == nil`) + and companion sub-processes such as a page store used by the shard. """ use GenServer diff --git a/shard/lib/net/addr.ex b/shard/lib/net/addr.ex index c1d2f05..b92ae70 100644 --- a/shard/lib/net/addr.ex +++ b/shard/lib/net/addr.ex @@ -1,4 +1,10 @@ defmodule SNet.Addr do + @moduledoc""" + Helper module for getting our IP addresses. + + Runs an agent that gets our public IPv4 address on the internet and stores it. + """ + use Agent require Logger @@ -21,6 +27,9 @@ defmodule SNet.Addr do end end + @doc""" + Reteurn the list of IPv4 address for our network interfaces. + """ def get_if_inet4 do {:ok, ifs} = :inet.getifaddrs for {_, opts} <- ifs, @@ -32,15 +41,24 @@ defmodule SNet.Addr do end end + @doc""" + Return our public IPv4 address as observed by an external API provider (`ipify.org`) + """ def get_pub_inet4 do Agent.get(__MODULE__, &(&1)) end + @doc""" + Get all our IPv4 addresses. + """ def get_all_inet4 do addrset = for x <- get_if_inet4() ++ get_pub_inet4(), into: %MapSet{}, do: x MapSet.to_list addrset end + @doc""" + Determines if an IP address is ours or not. + """ def is_local?({:inet, ip, port}) do port == Application.get_env(:shard, :port) and (ip == {127,0,0,1} or ip in get_all_inet4()) end diff --git a/shard/lib/net/auth.ex b/shard/lib/net/auth.ex index c903093..186b506 100644 --- a/shard/lib/net/auth.ex +++ b/shard/lib/net/auth.ex @@ -1,3 +1,13 @@ defmodule SNet.Auth do + @moduledoc""" + Structure for auth values that define if a connection is with an anonymous + peer or with an authenticated peer. + + Message handlers in shards will receive an `auth` parameter equal to `nil` if the + connection where the message comes from is not authenticated, or + `%SNet.Auth{my_pk: my_pk, his_pk: his_pk}` in the case where the connection is authenticated, + we are known to them as `my_pk` and they are known to us as `his_pk`. + """ + defstruct [:my_pk, :his_pk] end diff --git a/shard/lib/net/group.ex b/shard/lib/net/group.ex index a5f0867..f3d5962 100644 --- a/shard/lib/net/group.ex +++ b/shard/lib/net/group.ex @@ -25,11 +25,17 @@ defprotocol SNet.Group do @doc""" Check if a peer is allowed to participate in this group. + The `auth` parameter is `nil` or a `SNet.Auth` struct. """ def in_group?(group, conn_pid, auth) end defmodule SNet.PubShardGroup do + @moduledoc""" + A network group defined as all the people interested in a given shard. + + %SNet.PubShardGroup{id: shard_id} + """ defstruct [:id] defimpl SNet.Group do @@ -85,6 +91,12 @@ defmodule SNet.PubShardGroup do end defmodule SNet.PrivGroup do + @moduledoc""" + A private networking group defined by the list of public keys of people allowed to + participate. + + %SNet.PrivGroup{pk_list: [pk1, pk2, ...]} + """ defstruct [:pk_list] defimpl SNet.Group do diff --git a/shard/lib/net/manager.ex b/shard/lib/net/manager.ex index fb92f13..e4d8ad9 100644 --- a/shard/lib/net/manager.ex +++ b/shard/lib/net/manager.ex @@ -1,9 +1,8 @@ defmodule SNet.Manager do @moduledoc""" - - :connections (not persistent) + Maintains a table `:connections` of currently connected peers, which is a list of: - List of - { peer_info, pid, nil | {my_pk, his_pk} } + { peer_info, pid, nil | %SNet.Auth{my_pk: my_pk, his_pk: his_pk} } """ use GenServer @@ -91,7 +90,7 @@ defmodule SNet.Manager do @doc""" Connect to a peer specified by ip address and port - peer_info := {:inet, ip, port} + peer_info := {:inet, ip, port} """ def add_peer(peer_info, opts \\ []) do GenServer.call(__MODULE__, {:add_peer, peer_info, opts[:auth], opts[:callback]}) @@ -134,6 +133,11 @@ defmodule SNet.Manager do end end + @doc""" + Send message to a peer specified by peer info over authenticated channel. + `auth` is a `SNet.Auth` struct describing the required authentication. + Opens a connection if necessary. + """ def send_auth(peer_info, auth, msg) do case :ets.match(:connections, {peer_info, :'$1', auth, :_}) do [[pid]|_] -> diff --git a/shard/lib/net/tcpconn.ex b/shard/lib/net/tcpconn.ex index 21d25df..dc33bff 100644 --- a/shard/lib/net/tcpconn.ex +++ b/shard/lib/net/tcpconn.ex @@ -1,10 +1,9 @@ defmodule SNet.TCPConn do @moduledoc""" Secret handshake as described in this document: - https://ssbc.github.io/scuttlebutt-protocol-guide/#peer-connections + <https://ssbc.github.io/scuttlebutt-protocol-guide/#peer-connections> - Does not implement the stream protocol, we don't hide the length of packets. - (TODO ^) + TODO: Does not implement the stream protocol, we don't hide the length of packets. """ @@ -18,15 +17,15 @@ defmodule SNet.TCPConn do Expected initial state: a dict with the following keys: - - socket: the socket - - is_client: true if we are the initiator of the connection, false otherwise - - my_port: if we are the client, what port should the other dial to recontact us + - `socket`: the socket, if we are responding to a connection + - `connect_to`: the IP and port we want to connect to, if we are the initiator of the connection + - `my_port`: if we are the initiator, what port should the other dial to recontact us Optionnally, and only if we are the initiator of the connection, the following key: - - auth: nil | {my_pk, list_accepted_his_pk} + - `auth`: `nil | {my_pk, list_accepted_his_pk}` - If we are initiator of the connection, we will use crypto if and only if auth is not nil. + If we are initiator of the connection, we will use crypto if and only if auth is not `nil`. """ def start_link(state) do GenServer.start_link(__MODULE__, state) diff --git a/shard/lib/net/tcpserver.ex b/shard/lib/net/tcpserver.ex index d7326ad..827fefa 100644 --- a/shard/lib/net/tcpserver.ex +++ b/shard/lib/net/tcpserver.ex @@ -1,4 +1,8 @@ defmodule SNet.TCPServer do + @moduledoc""" + Process for accepting TCP connections from peers. + """ + require Logger use Task, restart: :permanent diff --git a/shard/lib/shard_uri.ex b/shard/lib/shard_uri.ex index 1b186d2..81b4ab1 100644 --- a/shard/lib/shard_uri.ex +++ b/shard/lib/shard_uri.ex @@ -1,8 +1,12 @@ defmodule ShardURI do @moduledoc""" - Convert Shard manifests to and from text strings. + Convert Shard manifests to and from text strings. Not used by the shard library + internally, only provided for convenience. """ + @doc""" + Get URI corresponding to shard manifest. + """ def from_manifest(m) do case m do %SApp.Chat.Manifest{channel: chan} -> "shard:chat:#{chan}" @@ -19,6 +23,9 @@ defmodule ShardURI do end end + @doc""" + Parse URI and return corresponding manifest. + """ def to_manifest(p) do case p do "shard:chat:" <> chan -> |