Document

22 files changed, 295 insertions, 109 deletions

diff --git a/README.md b/README.md
index c247e4e..db52b04 100644
--- a/README.md
+++ b/README.md
@@ -86,6 +86,8 @@ Current status
 What is available
 -----------------
 
+All of these are rudimentary prototypes at an early stage.
+
 * Chat rooms (public and private) with full history and efficient data structure for
   retrieving missing messages after disconnection
 * File upload (public only)
@@ -99,6 +101,8 @@ See `TODO` file for more details.
 * Finding peers via DHT (very easy to add)
 * Invite/notification system
 * Good access control
+* Good networking behind NAT/Firewall
+* Automatic discovery on local networks
 * More applications
 
 How to use it?
@@ -217,4 +221,5 @@ This CLI supports a few basic commands:
 - `/pm nickname1 [nickname2] [...]`: enter private conversation with someone
 - `/send_file path`: make file available on the network and send link to current chat room.
   **WARNING: all files are publicly available for now, even if they are sent in a private chat room.**
+- `/shards`: return the list of all shards on the system
 - `/quit`: return to iex prompt
diff --git a/shard/lib/app/chat.ex b/shard/lib/app/chat.ex
index ff0c97d..b046fc3 100644
--- a/shard/lib/app/chat.ex
+++ b/shard/lib/app/chat.ex
@@ -12,11 +12,7 @@ defmodule SApp.Chat do
       %SApp.Chat.PrivChat.Manifest{pk_list: ordered_list_of_authorized_pks}
 
   Future improvements:
-  - message signing
-  - storage of the chatroom messages to disk
   - use a DHT to find peers that are interested in this channel
-  - epidemic broadcast (carefull not to be too costly,
-    maybe by limiting the number of peers we talk to)
   - partial synchronization only == data distributed over peers
   """
 
@@ -32,7 +28,9 @@ defmodule SApp.Chat do
 
   defmodule Manifest do
     @moduledoc"""
-    Manifest for a public chat room defined by its name.
+    Manifest for a public chat room defined by its name. Example:
+
+        %SApp.Chat.Manifest{channel: "test"}
     """
 
     defstruct [:channel]
@@ -73,19 +71,22 @@ defmodule SApp.Chat do
   # ==========
 
   defmodule State do
+    @moduledoc"""
+    Internal state struct of chat shard.
+    """
+
     defstruct [:id, :netgroup, :manifest, :page_store, :mst, :subs, :read]
   end
   
   @doc """
-  Start a process that connects to a given channel
+  Start a process that connects to a given channel. Don't call directly, use for instance:
+
+      Shard.Manager.find_or_start %SApp.Chat.Manifest{channel: "my_chan"}
   """
   def start_link(manifest) do
     GenServer.start_link(__MODULE__, manifest)
   end
 
-  @doc """
-  Initialize channel process.
-  """
   def init(manifest) do
     id = SData.term_hash manifest
 
@@ -103,7 +104,7 @@ defmodule SApp.Chat do
     end
     root = cond do
       root == nil -> nil
-      GenServer.call(page_store, {:have_rec, root}) -> root
+      SApp.PageStore.have_rec?(page_store, root) -> root
       true ->
         Logger.warn "Not all pages for saved root were saved, restarting from an empty state!"
         nil
@@ -124,9 +125,6 @@ defmodule SApp.Chat do
     }
   end
 
-  @doc """
-  Implementation of the :manifest call that returns the chat room's manifest
-  """
   def handle_call(:manifest, _from, state) do
     {:reply, state.manifest, state}
   end
@@ -166,11 +164,6 @@ defmodule SApp.Chat do
     {:noreply, state}
   end
 
-  @doc """
-  Implementation of the :chat_send handler. This is the main handler that is used
-  to send a message to the chat room. Puts the message in the store and syncs
-  with all connected peers.
-  """
   def handle_cast({:chat_send, pk, msg}, state) do
     next_ts = case MST.last(state.mst, nil, 1) do
       [] -> System.os_time :seconds
@@ -204,10 +197,6 @@ defmodule SApp.Chat do
     {:noreply, state}
   end
 
-  @doc """
-  Implementation of the :interested handler, this is called when a peer we are
-  connected to asks to recieve data for this channel.
-  """
   def handle_cast({:interested, conn_pid, auth}, state) do
     if SNet.Group.in_group?(state.netgroup, conn_pid, auth) do
       SNet.Manager.send_pid(conn_pid, {state.id, nil, {:root, state.mst.root, true}})
@@ -221,10 +210,6 @@ defmodule SApp.Chat do
     {:noreply, %{ state | subs: new_subs }}
   end
 
-  @doc """
-  Implementation of the :msg handler, which is the main handler for messages
-  comming from other peers concerning this chat room.
-  """
   def handle_cast({:msg, conn_pid, auth, _shard_id, nil, msg}, state) do
     if not SNet.Group.in_group?(state.netgroup, conn_pid, auth) do
       # Ignore message
@@ -245,7 +230,7 @@ defmodule SApp.Chat do
                 mst2 = MST.insert(state.mst, msgitem)
                 if mst2.root == new_root do
                   state = %{state | mst: mst2}
-                  GenServer.cast(state.page_store, {:set_roots, [mst2.root]})
+                  SApp.PageStore.set_roots(state.page_store, [mst2.root])
                   save_state(state)
                   msg_callback(state, msgitem)
                   SNet.Group.broadcast(state.netgroup, {state.id, nil, msg}, exclude_pid: [conn_pid])
@@ -311,7 +296,7 @@ defmodule SApp.Chat do
         for x <- new do
           msg_callback(state, x)
         end
-        GenServer.cast(state.page_store, {:set_roots, [mst.root]})
+        SApp.PageStore.set_roots(state.page_store, [mst.root])
         state = %{state | mst: mst}
         save_state(state)
         if state.mst.root != old_root do
@@ -365,15 +350,15 @@ defmodule SApp.Chat do
 
   The process calling this function will start recieving messages of the form:
 
-    {:chat_recv, manifest, {pk, msgbin, sign}}
+      {:chat_recv, manifest, {pk, msgbin, sign}}
 
   or
 
-    {:chat_send, manifest, {pk, msgbin, sign}}
+      {:chat_send, manifest, {pk, msgbin, sign}}
 
   msgbin can be used in the following way:
 
-    {timestamp, message} = SData.term_unbin msgbin
+      {timestamp, message} = SData.term_unbin msgbin
   """
   def subscribe(shard_pid) do
     GenServer.cast(shard_pid, {:subscribe, self()})
diff --git a/shard/lib/app/directory.ex b/shard/lib/app/directory.ex
index cbea8c3..257e8b9 100644
--- a/shard/lib/app/directory.ex
+++ b/shard/lib/app/directory.ex
@@ -29,9 +29,18 @@ defmodule SApp.Directory do
   end
 
   defmodule State do
+    @moduledoc"""
+    Internal state struct of directory shard.
+    """
+
     defstruct [:owner, :public, :name, :manifest, :id, :netgroup, :items, :revitems]
   end
 
+  @doc"""
+  Start a process that connects to a given channel. Don't call directly, use for instance:
+
+      Shard.Manager.find_or_start %SApp.Directory.Manifest{owner: my_pk, public: false, name: "collection"}
+  """
   def start_link(manifest) do
     GenServer.start_link(__MODULE__, manifest)
   end
@@ -217,23 +226,23 @@ defmodule SApp.Directory do
   @doc"""
   Return list of items stored in this directory.
 
-  Returns a dictionnary of %{name => {manifest, stored?}}.
+  Returns a dictionnary of `%{name => {manifest, stored?}}`.
   """
   def get_items(pid) do
     GenServer.call(pid, :get_items)
   end
 
   @doc"""
-  Return the manifest of item with a given name in directory, or nil if not found.
+  Return the manifest of item with a given name in directory, or `nil` if not found.
   
-  Equivalent to get_items(pid)[name] but better.
+  Equivalent to `get_items(pid)[name]` but better.
   """
   def read(pid, name) do
     GenServer.call(pid, {:read, name})
   end
 
   @doc"""
-  Find an item in the directory by its manifest. Returns name if found or nil if not found.
+  Find an item in the directory by its manifest. Returns name if found or `nil` if not found.
   """
   def find(pid, manifest) do
     GenServer.call(pid, {:find, manifest})
@@ -241,8 +250,8 @@ defmodule SApp.Directory do
 
   @doc"""
   Add an item to this directory. An item is a name for a shard manifest.
-  An item added to a directory becomes a dependency of the directory, i.e.
-  if the directory is pinned then all items inside are pinned as well.
+  An item added to a directory with `stored = true` becomes a dependency of the directory,
+  i.e. if the directory is pinned then all items inside are pinned as well.
   """
   def add_item(pid, name, manifest, stored \\ true) do
     GenServer.call(pid, {:add_item, name, manifest, stored})
diff --git a/shard/lib/app/file.ex b/shard/lib/app/file.ex
index e2a9798..0e07cc3 100644
--- a/shard/lib/app/file.ex
+++ b/shard/lib/app/file.ex
@@ -9,9 +9,12 @@ defmodule SApp.File do
         file_hash: hash
         size: int
         mime_type: string
-    }
+      }
+
+  The file is cut in blocks that are collected in a k-ary Merkle tree
+  (see SData.MerkleTree for block size and k value).
 
-  The file is cut in blocks of 4kb that are collected in a 64-ary Merkle tree.
+  TODO I feel bad about some parts of the logic in here.
   """
 
   use GenServer
@@ -26,7 +29,7 @@ defmodule SApp.File do
   defmodule Manifest do
     @moduledoc"""
     Manifest for a file.
-    The file is identified by the root hash of its Merkle tree and by its mime type.
+    The file is identified by its infohash, which is the hash of a `SApp.File.Info` struct.
     """
     defstruct [:infohash]
 
@@ -46,9 +49,21 @@ defmodule SApp.File do
   end
 
   defmodule State do
+    @moduledoc"""
+    Internal state struct for file shard.
+    """
     defstruct [:infohash, :id, :manifest, :netgroup, :info, :infobin, :store, :missing, :path, :reqs]
   end
 
+  @doc """
+  Start a process that connects to a given channel. Don't call directly, use for instance:
+
+      Shard.Manager.find_or_start %SApp.File.Manifest{infohash: "some_infohash"}
+
+  or:
+
+      SApp.File.Create("/path/to/file", "mime/type")
+  """
   def start_link(manifest) do
     GenServer.start_link(__MODULE__, manifest)
   end  
@@ -229,7 +244,7 @@ defmodule SApp.File do
         true ->
           meta = get_mt(state)
           n_blocks = MT.block_count(meta)
-          expected_hashes = MT.get_range(meta, 0..(n_blocks-1))
+          expected_hashes = MT.get_all(meta)
           actual_hashes = if File.exists?(state.path) do
             File.stream!(state.path, [], MT.block_size())
             |> Enum.map(&(:crypto.hash(:sha256, &1)))
diff --git a/shard/lib/app/identity.ex b/shard/lib/app/identity.ex
index 78abbe7..7422822 100644
--- a/shard/lib/app/identity.ex
+++ b/shard/lib/app/identity.ex
@@ -34,9 +34,17 @@ defmodule SApp.Identity do
   end
 
   defmodule State do
+    @moduledoc"""
+    Internal state struct for identity shard.
+    """
     defstruct [:pk, :id, :state, :netgroup]
   end
 
+  @doc """
+  Start a process that connects to a given channel. Don't call directly, use for instance:
+
+      Shard.Manager.find_or_start %SApp.Identity.Manifest{pk: some_public_key}
+  """
   def start_link(manifest) do
     GenServer.start_link(__MODULE__, manifest)
   end
diff --git a/shard/lib/app/pagestore.ex b/shard/lib/app/pagestore.ex
index 3cda51d..0cbb10a 100644
--- a/shard/lib/app/pagestore.ex
+++ b/shard/lib/app/pagestore.ex
@@ -7,12 +7,29 @@ defmodule SApp.PageStore do
 
   Uses an ETS table of:
   
-    { page_id, why_have_it }         -- waiting for data
-    { page_id, why_have_it, data }   -- once we have the data
+      { page_id, why_have_it }         # waiting for data
+      { page_id, why_have_it, data }   # once we have the data
 
-  why_have_it := :root
-               | {:req_by, some_other_page_id}
-               | {:cached, expiry_date}
+      why_have_it := :root
+                   | {:req_by, some_other_page_id}
+                   | {:cached, expiry_date}
+
+  TODO: at the moment we are trying to pull all missing pages at once from our peers.
+  This can work for metadata that isn't too big but won't work with bigger objects.
+  Have a smart strategy where we limit the number of requests currently in-flight but
+  still make sure everything gets pulled in. This will also pave the way to selectively
+  pulling in pages, for instance if we have a function to give them a priority score and
+  a maximum stored page count.
+
+  A `SApp.PageStore` can be used as a `SData.PageStore` in the following way:
+
+      %SApp.PageStore{pid: store_pid}
+
+  or:
+
+      %SApp.PageStore{pid: store_pid, prefer_ask: [connection_pid, ...]}
+
+  In the second case, missing pages will be requested first to the specified peers.
   """
 
   use GenServer
@@ -25,6 +42,9 @@ defmodule SApp.PageStore do
   @max_failures 4         # Maximum of peers that reply not_found before we abandon
 
   defmodule State do
+    @moduledoc"""
+    Internal state struct of pagestore process.
+    """
     defstruct [:shard_id, :path, :netgroup, :store, :reqs, :retries, :store_path]
   end
 
@@ -258,7 +278,7 @@ defmodule SApp.PageStore do
     {:noreply, state}
   end
 
-  def ask_random_peers(state, key) do
+  defp ask_random_peers(state, key) do
     SNet.Group.broadcast(state.netgroup, {state.shard_id, state.path, {:get, key}}, nmax: 3)
   end
 
@@ -289,4 +309,24 @@ defmodule SApp.PageStore do
       store ## DO SOMETHING???
     end
   end
+
+  # ====================
+  # PAGE STORE INTERFACE
+  # ====================
+
+  @doc"""
+  Returns `true` if the page store currently stores the specified root page
+  and all its dependencies, recursively.
+  """
+  def have_rec?(pid, root) do
+    GenServer.call(pid, {:have_rec, root})
+  end
+
+  @doc"""
+  Define the set of root pages we are interested in. This will start pulling in
+  the defined pages and all their dependencies recursively if we don't have them.
+  """
+  def set_roots(pid, roots) do
+    GenServer.cast(pid, {:set_roots, roots})
+  end
 end
diff --git a/shard/lib/cli/cli.ex b/shard/lib/cli/cli.ex
index 54b882f..8495b93 100644
--- a/shard/lib/cli/cli.ex
+++ b/shard/lib/cli/cli.ex
@@ -1,12 +1,23 @@
 defmodule SCLI do
   @moduledoc """
-  Small command line interface for the chat application
+  Small command line interface for the chat application. Supports public chat rooms,
+  private conversations, sending files (but not receiving them - could be done easily).
+
+  The code of this module is intended as an example of how to use the Shard library.
+
+  TODO: more commands.
   """
 
   defmodule State do
+    @moduledoc"""
+    Internal state struct of the CLI.
+    """
     defstruct [:room_pid, :id_pid, :pk]
   end
 
+  @doc"""
+  Call this from the iex prompt to launch the CLI.
+  """
   def run() do
     for {_chid, manifest, _} <- Shard.Manager.list_shards do
       case manifest do
@@ -197,6 +208,14 @@ defmodule SCLI do
     state
   end
 
+  defp handle_command(state, ["shards"]) do
+    Shard.Manager.list_shards
+    |> Enum.map(&(ShardURI.from_manifest(elem(&1, 1))))
+    |> Enum.sort()
+    |> Enum.map(&IO.puts/1)
+    state
+  end
+
   defp handle_command(state, _cmd) do
     IO.puts "Invalid command"
     state
diff --git a/shard/lib/data/data.ex b/shard/lib/data/data.ex
index 33dca09..8d2b277 100644
--- a/shard/lib/data/data.ex
+++ b/shard/lib/data/data.ex
@@ -10,34 +10,46 @@ defmodule SData do
   These functions must only return :duplicate for equal items.
   """
 
-  @doc """
+  @doc"""
   Calculate the hash of an Erlang term by first converting it to its
-  binary representation.
+  binary representation. Equivalent to `bin_hash(term_bin(term))`.
   """
   def term_hash(term, algo \\ :sha256) do
     :crypto.hash(algo, (:erlang.term_to_binary term))
   end
 
+  @doc"""
+  Convert any Erlang term to a binary representation.
+  """
   def term_bin(term) do
   	:erlang.term_to_binary term
   end
 
+  @doc"""
+  Calculate the hash of a binary.
+  """
   def bin_hash(bin, algo \\ :sha256) do
     :crypto.hash(algo, bin)
   end
 
+  @doc"""
+  Calculate the hash of a file.
+  """
   def file_hash(path, algo \\ :sha256) do
     File.stream!(path, [], 65536)
     |> Enum.reduce(:crypto.hash_init(algo), &(:crypto.hash_update(&2, &1)))
     |> :crypto.hash_final()
   end
 
+  @doc"""
+  Recover an Erlang term from its binary representation.
+  """
   def term_unbin(bin) do
     :erlang.binary_to_term(bin, [:safe])
   end
 
   @doc"""
-    Compare function for arbitrary terms using the Erlang order
+  Compare function for arbitrary terms using the Erlang order
   """
   def cmp_term(a, b) do
     cond do
@@ -48,7 +60,7 @@ defmodule SData do
   end
 
   @doc"""
-    Compare function for timestamped strings
+  Compare function for timestamped strings
   """
   def cmp_ts_str({ts1, str1}, {ts2, str2}) do
     cond do
@@ -61,7 +73,7 @@ defmodule SData do
   end
 
   @doc"""
-    Merge function for nils
+  Merge function for nils
   """
   def merge_true(true, true), do: true
 end
diff --git a/shard/lib/data/merklelist.ex b/shard/lib/data/merklelist.ex
index 9b44ee8..c450ca7 100644
--- a/shard/lib/data/merklelist.ex
+++ b/shard/lib/data/merklelist.ex
@@ -1,12 +1,8 @@
 defmodule SData.MerkleList do
   @moduledoc"""
-  A simple Merkle list store.
+  A simple Merkle list store. Not used.
 
-  Future improvements:
-    - When messages are inserted other than at the top, all intermediate hashes
-      change. Keep track of the mapping from old hashes to new hashes so that get
-      requests can work even for hashes that are not valid anymore.
-    - group items in "pages" (bigger bundles)
+  TODO delete this module
   """
 
   defstruct [:root, :top, :cmp, :store]
diff --git a/shard/lib/data/merklesearchtree.ex b/shard/lib/data/merklesearchtree.ex
index e646774..f67843d 100644
--- a/shard/lib/data/merklesearchtree.ex
+++ b/shard/lib/data/merklesearchtree.ex
@@ -3,15 +3,15 @@ defmodule SData.MerkleSearchTree do
   A Merkle search tree.
 
   A node of the tree is
-    {
-      level,
-      hash_of_node | nil,
-      [ 
-        { item_low_bound, hash_of_node | nil },
-        { item_low_bound, hash_of_node | nil },
-        ...
+      {
+        level,
+        hash_of_node | nil,
+        [ 
+          { item_low_bound, hash_of_node | nil },
+          { item_low_bound, hash_of_node | nil },
+          ...
+        }
       }
-    }
   """
 
   alias SData.PageStore, as: Store
diff --git a/shard/lib/data/merkletree.ex b/shard/lib/data/merkletree.ex
index 73679cf..94bd443 100644
--- a/shard/lib/data/merkletree.ex
+++ b/shard/lib/data/merkletree.ex
@@ -92,9 +92,17 @@ defmodule SData.MerkleTree do
   end
 
   @doc"""
-  Get the hashes of all blocks in a range
+  Get the hashes of all blocks
   """
-  def get_range(mt, range) do
-    range |> Enum.map(&(get(mt, &1))) # TODO: do this efficiently
+  def get_all(mt) do
+    %Page{child_nblk: cn, list: list} = Store.get(mt.store, mt.root)
+    if cn == 1 do
+      list
+    else
+      list
+      |> Enum.map(&(%{mt | root: &1}))
+      |> Enum.map(&get_all/1)
+      |> Enum.reduce([], &(&2++&1))
+    end
   end
 end
diff --git a/shard/lib/data/signrev.ex b/shard/lib/data/signrev.ex
index 6360b53..164df03 100644
--- a/shard/lib/data/signrev.ex
+++ b/shard/lib/data/signrev.ex
@@ -56,7 +56,7 @@ defmodule SData.SignRev do
   @doc"""
   Check that a signed binary is correct and merge it into the SignRev.
 
-  Returns {true, new_sr} if an update happenned, {false, sr} otherwise.
+  Returns `{true, new_sr}` if an update happenned, `{false, sr}` otherwise.
   """
   def merge(sr, signed, pk) do
     case Shard.Keys.open(pk, signed) do
diff --git a/shard/lib/data/store.ex b/shard/lib/data/store.ex
index ca12cd0..ce5618c 100644
--- a/shard/lib/data/store.ex
+++ b/shard/lib/data/store.ex
@@ -24,6 +24,10 @@ defprotocol SData.PageStore do
 
   This protocol may also be implemented by store proxies that track
   operations and implement different synchronization or caching mechanisms.
+
+  A page store is an object that stores data pages (arbitrary Erlang terms) and
+  identifies them by their hash. Dependencies may exist between pages, in which
+  case they form a Merkle DAG.
   """
 
   @doc"""
@@ -60,8 +64,15 @@ end
 
 
 defmodule SData.LocalStore do
+  @moduledoc"""
+  A page store that saves all pages locally in RAM. The store is basically a dictionnary
+  of hash to term mappings, which is mutated by put operations.
+  """
   defstruct [:pages] 
 
+  @doc"""
+  Create empty LocalStore.
+  """
   def new() do
     %SData.LocalStore{ pages: %{} }
   end
diff --git a/shard/lib/keys.ex b/shard/lib/keys.ex
index 412baa2..3a97b5f 100644
--- a/shard/lib/keys.ex
+++ b/shard/lib/keys.ex
@@ -1,6 +1,6 @@
 defmodule Shard.Keys do
   @moduledoc"""
-    Module for saving private keys.
+  Module for saving private keys, signing messages and checking message signatures.
   """
 
   use Agent
@@ -39,6 +39,10 @@ defmodule Shard.Keys do
     :binary.longest_common_suffix([pk, suffix]) == byte_size(suffix)
   end
 
+  @doc"""
+  Return any public key for which we have the secret key. Generates a new keypair
+  if necessary.
+  """
   def get_any_identity() do
     Agent.get(__MODULE__, fn _ ->
       case list_identities() do
@@ -96,6 +100,9 @@ defmodule Shard.Keys do
     end
   end
 
+  @doc"""
+  Check if we have the secret key associated with a public key.
+  """
   def have_sk?(pk) do
     case :dets.lookup @key_db, pk do
       [{^pk, _sk}] -> true
@@ -103,6 +110,9 @@ defmodule Shard.Keys do
     end
   end
 
+  @doc"""
+  Return the secret key associated with a public key if we have it or `nil` otherwise.
+  """
   def get_sk(pk) do
     case :dets.lookup @key_db, pk do
       [{^pk, sk}] -> sk
@@ -111,12 +121,12 @@ defmodule Shard.Keys do
   end
 
   @doc"""
-    Lookup the secret key for a pk and generate a detached signature for a message.
+  Lookup the secret key for a pk and generate a detached signature for a message.
 
-    The original message is not returned.
+  The original message is not returned.
 
-    Answer is {:ok, signature} if it worked, or :not_found if we didn't find the key.
-    
+  Answer is {:ok, signature} if it worked, or :not_found if we don't have the corresponding
+  secret key.
   """
   def sign_detached(pk, bin) do
     case :dets.lookup @key_db, pk do
@@ -127,9 +137,9 @@ defmodule Shard.Keys do
   end
 
   @doc"""
-    Verify a detached signature for a message
+  Verify a detached signature for a message
 
-    Returns :ok if the signature was correct.
+  Returns :ok if the signature was correct.
   """
   def verify(pk, bin, sign) do
     if valid_identity_pk? pk do
@@ -143,12 +153,16 @@ defmodule Shard.Keys do
   end
 
   @doc"""
-    Check if a public key is a valid identity pk. Requirement: have the correct suffix.
+  Check if a public key is a valid identity pk. Requirement: have the correct suffix.
   """
   def valid_identity_pk?(pk) do
     check_suffix(pk, Application.get_env(:shard, :identity_suffix))
   end
 
+  @doc"""
+  Creates a displayable representation of a public key by taking the hex representation
+  of its first four bytes. (not tamper proof but better than nothing)
+  """
   def pk_display(pk) do
     pk
     |> binary_part(0, 4)
diff --git a/shard/lib/manager.ex b/shard/lib/manager.ex
index c3897a3..ed21380 100644
--- a/shard/lib/manager.ex
+++ b/shard/lib/manager.ex
@@ -14,7 +14,8 @@ defprotocol Shard.Manifest do
   """
 
   @doc"""
-  Get the module in question.
+  Get the module that implements the shard. All shard modules must have a function
+  `start_link` that start the shard process and take a single argument: the manifest.
   """
   def module(manifest)
 
@@ -26,29 +27,38 @@ end
 
 defmodule Shard.Manager do
   @moduledoc"""
-    Maintains several important tables :
+  The manager is the main process by which shards are started, stopped, and their lifetime
+  monitored.
 
-    - :shard_db (persistent with DETS)
-    
-      List of
-        { id, manifest, why_have_it, state }
+  Maintains several important tables :
 
-      why_have_it := {:pinned, %MapSet{who requires it...}, %MapSet{who it requires...}}
-                   | {:req, %MapSet{who requires it...}, %MapSet{who it requires...}}
-                   | {:cached, expiry_date}
+  - `@shard_db` (persistent with DETS), a list of:
+  
+    ```
+    { id, manifest, why_have_it, state }
+
+    why_have_it := {:pinned, %MapSet{who requires it...}, %MapSet{who it requires...}}
+                 | {:req, %MapSet{who requires it...}, %MapSet{who it requires...}}
+                 | {:cached, expiry_date}
+    ```
+
+  - `@peer_db` (persistent with DETS), a multi-list of:
 
-    - :peer_db (persistent with DETS)
+    ```
+    { shard_id, peer_info }   # TODO: add health info (last seen, ping, etc)
 
-      Mult-list of
-        { shard_id, peer_info }   # TODO: add health info (last seen, ping, etc)
+    peer_info := {:inet, ip, port}
+    TODO peer_info |= {:inet6, ip, port} | {:onion, name}
+  ```
 
-      peer_info := {:inet, ip, port}
-      TODO peer_info |= {:inet6, ip, port} | {:onion, name}
+  - `:shard_procs` (not persistent), a list of:
 
-    - :shard_procs (not persistent)
+    ```
+    { {id, path}, pid }
+    ```
 
-      List of
-        { {id, path}, pid }
+    The path value is used to distinguish between a shard's main process (`path == nil`)
+    and companion sub-processes such as a page store used by the shard.
   """
 
   use GenServer
diff --git a/shard/lib/net/addr.ex b/shard/lib/net/addr.ex
index c1d2f05..b92ae70 100644
--- a/shard/lib/net/addr.ex
+++ b/shard/lib/net/addr.ex
@@ -1,4 +1,10 @@
 defmodule SNet.Addr do
+  @moduledoc"""
+  Helper module for getting our IP addresses.
+
+  Runs an agent that gets our public IPv4 address on the internet and stores it.
+  """
+
   use Agent
 
   require Logger
@@ -21,6 +27,9 @@ defmodule SNet.Addr do
     end
   end
 
+  @doc"""
+  Reteurn the list of IPv4 address for our network interfaces.
+  """
   def get_if_inet4 do
     {:ok, ifs} = :inet.getifaddrs
     for {_, opts} <- ifs,
@@ -32,15 +41,24 @@ defmodule SNet.Addr do
     end
   end
 
+  @doc"""
+  Return our public IPv4 address as observed by an external API provider (`ipify.org`)
+  """
   def get_pub_inet4 do
     Agent.get(__MODULE__, &(&1))
   end
 
+  @doc"""
+  Get all our IPv4 addresses.
+  """
   def get_all_inet4 do
     addrset = for x <- get_if_inet4() ++ get_pub_inet4(), into: %MapSet{}, do: x
     MapSet.to_list addrset
   end
 
+  @doc"""
+  Determines if an IP address is ours or not.
+  """
   def is_local?({:inet, ip, port}) do
     port == Application.get_env(:shard, :port) and (ip == {127,0,0,1} or ip in get_all_inet4())
   end
diff --git a/shard/lib/net/auth.ex b/shard/lib/net/auth.ex
index c903093..186b506 100644
--- a/shard/lib/net/auth.ex
+++ b/shard/lib/net/auth.ex
@@ -1,3 +1,13 @@
 defmodule SNet.Auth do
+  @moduledoc"""
+  Structure for auth values that define if a connection is with an anonymous
+  peer or with an authenticated peer.
+
+  Message handlers in shards will receive an `auth` parameter equal to `nil` if the
+  connection where the message comes from is not authenticated, or
+  `%SNet.Auth{my_pk: my_pk, his_pk: his_pk}` in the case where the connection is authenticated,
+  we are known to them as `my_pk` and they are known to us as `his_pk`.
+  """
+
   defstruct [:my_pk, :his_pk]
 end
diff --git a/shard/lib/net/group.ex b/shard/lib/net/group.ex
index a5f0867..f3d5962 100644
--- a/shard/lib/net/group.ex
+++ b/shard/lib/net/group.ex
@@ -25,11 +25,17 @@ defprotocol SNet.Group do
 
   @doc"""
   Check if a peer is allowed to participate in this group.
+  The `auth` parameter is `nil` or a `SNet.Auth` struct.
   """
   def in_group?(group, conn_pid, auth)
 end
 
 defmodule SNet.PubShardGroup do
+  @moduledoc"""
+  A network group defined as all the people interested in a given shard.
+
+      %SNet.PubShardGroup{id: shard_id}
+  """
   defstruct [:id]
 
   defimpl SNet.Group do
@@ -85,6 +91,12 @@ defmodule SNet.PubShardGroup do
 end
 
 defmodule SNet.PrivGroup do
+  @moduledoc"""
+  A private networking group defined by the list of public keys of people allowed to
+  participate.
+
+      %SNet.PrivGroup{pk_list: [pk1, pk2, ...]}
+  """
   defstruct [:pk_list]
 
   defimpl SNet.Group do
diff --git a/shard/lib/net/manager.ex b/shard/lib/net/manager.ex
index fb92f13..e4d8ad9 100644
--- a/shard/lib/net/manager.ex
+++ b/shard/lib/net/manager.ex
@@ -1,9 +1,8 @@
 defmodule SNet.Manager do
   @moduledoc"""
-  - :connections (not persistent)
+  Maintains a table `:connections` of currently connected peers, which is a list of:
 
-      List of
-        { peer_info, pid, nil | {my_pk, his_pk} }
+      { peer_info, pid, nil | %SNet.Auth{my_pk: my_pk, his_pk: his_pk} }
   """
 
   use GenServer
@@ -91,7 +90,7 @@ defmodule SNet.Manager do
   @doc"""
   Connect to a peer specified by ip address and port
 
-    peer_info := {:inet, ip, port}
+      peer_info := {:inet, ip, port}
   """
   def add_peer(peer_info, opts \\ []) do
     GenServer.call(__MODULE__, {:add_peer, peer_info, opts[:auth], opts[:callback]})
@@ -134,6 +133,11 @@ defmodule SNet.Manager do
     end
   end
 
+  @doc"""
+  Send message to a peer specified by peer info over authenticated channel.
+  `auth` is a `SNet.Auth` struct describing the required authentication.
+  Opens a connection if necessary.
+  """
   def send_auth(peer_info, auth, msg) do
     case :ets.match(:connections, {peer_info, :'$1', auth, :_}) do
       [[pid]|_] ->
diff --git a/shard/lib/net/tcpconn.ex b/shard/lib/net/tcpconn.ex
index 21d25df..dc33bff 100644
--- a/shard/lib/net/tcpconn.ex
+++ b/shard/lib/net/tcpconn.ex
@@ -1,10 +1,9 @@
 defmodule SNet.TCPConn do
   @moduledoc"""
   Secret handshake as described in this document:
-  https://ssbc.github.io/scuttlebutt-protocol-guide/#peer-connections
+  <https://ssbc.github.io/scuttlebutt-protocol-guide/#peer-connections>
 
-  Does not implement the stream protocol, we don't hide the length of packets.
-  (TODO ^)
+  TODO: Does not implement the stream protocol, we don't hide the length of packets.
   """
 
 
@@ -18,15 +17,15 @@ defmodule SNet.TCPConn do
 
   Expected initial state: a dict with the following keys:
 
-  - socket: the socket
-  - is_client: true if we are the initiator of the connection, false otherwise
-  - my_port: if we are the client, what port should the other dial to recontact us
+  - `socket`: the socket, if we are responding to a connection
+  - `connect_to`: the IP and port we want to connect to, if we are the initiator of the connection
+  - `my_port`: if we are the initiator, what port should the other dial to recontact us
 
   Optionnally, and only if we are the initiator of the connection, the following key:
 
-  - auth: nil | {my_pk, list_accepted_his_pk}
+  - `auth`: `nil | {my_pk, list_accepted_his_pk}`
 
-  If we are initiator of the connection, we will use crypto if and only if auth is not nil.
+  If we are initiator of the connection, we will use crypto if and only if auth is not `nil`.
   """
   def start_link(state) do
     GenServer.start_link(__MODULE__, state)
diff --git a/shard/lib/net/tcpserver.ex b/shard/lib/net/tcpserver.ex
index d7326ad..827fefa 100644
--- a/shard/lib/net/tcpserver.ex
+++ b/shard/lib/net/tcpserver.ex
@@ -1,4 +1,8 @@
 defmodule SNet.TCPServer do
+  @moduledoc"""
+  Process for accepting TCP connections from peers.
+  """
+
   require Logger
   use Task, restart: :permanent
 
diff --git a/shard/lib/shard_uri.ex b/shard/lib/shard_uri.ex
index 1b186d2..81b4ab1 100644
--- a/shard/lib/shard_uri.ex
+++ b/shard/lib/shard_uri.ex
@@ -1,8 +1,12 @@
 defmodule ShardURI do
   @moduledoc"""
-  Convert Shard manifests to and from text strings.
+  Convert Shard manifests to and from text strings. Not used by the shard library
+  internally, only provided for convenience.
   """
 
+  @doc"""
+  Get URI corresponding to shard manifest.
+  """
   def from_manifest(m) do
     case m do
       %SApp.Chat.Manifest{channel: chan} -> "shard:chat:#{chan}"
@@ -19,6 +23,9 @@ defmodule ShardURI do
     end
   end
 
+  @doc"""
+  Parse URI and return corresponding manifest.
+  """
   def to_manifest(p) do
     case p do
       "shard:chat:" <> chan ->