Compile with Nix

5 files changed, 708 insertions, 0 deletions

diff --git a/nix/builder/default.nix b/nix/builder/default.nix
new file mode 100644
index 0000000..8b8ae19
--- /dev/null
+++ b/nix/builder/default.nix
@@ -0,0 +1,387 @@
+{ stdenv
+, stdenvNoCC
+, runCommand
+, buildEnv
+, lib
+, fetchgit
+, removeReferencesTo
+, jq
+, cacert
+, pkgs
+, pkgsBuildBuild
+}:
+let
+
+  inherit (builtins) substring toJSON hasAttr trace split readFile elemAt;
+  inherit (lib)
+    concatStringsSep replaceStrings removePrefix optionalString pathExists
+    optional concatMapStrings fetchers filterAttrs mapAttrs mapAttrsToList
+    warnIf optionalAttrs platforms
+    ;
+
+  parseGoMod = import ./parser.nix;
+
+  removeExpr = refs: ''remove-references-to ${concatMapStrings (ref: " -t ${ref}") refs}'';
+
+  # Internal only build-time attributes
+  internal =
+    let
+      mkInternalPkg = name: src: pkgsBuildBuild.runCommand "gomod2nix-${name}"
+        {
+          inherit (pkgsBuildBuild.go) GOOS GOARCH;
+          nativeBuildInputs = [ pkgsBuildBuild.go ];
+        } ''
+        export HOME=$(mktemp -d)
+        cp ${src} src.go
+        go build -o $out src.go
+      '';
+    in
+    {
+
+      # Create a symlink tree of vendored sources
+      symlink = mkInternalPkg "symlink" ./symlink/symlink.go;
+
+      # Install development dependencies from tools.go
+      install = mkInternalPkg "symlink" ./install/install.go;
+
+    };
+
+  fetchGoModule =
+    { hash
+    , goPackagePath
+    , version
+    , go ? pkgs.go
+    }:
+    stdenvNoCC.mkDerivation {
+      name = "${baseNameOf goPackagePath}_${version}";
+      builder = ./fetch.sh;
+      inherit goPackagePath version;
+      nativeBuildInputs = [ go jq ];
+      outputHashMode = "recursive";
+      outputHashAlgo = null;
+      outputHash = hash;
+      SSL_CERT_FILE = "${cacert}/etc/ssl/certs/ca-bundle.crt";
+      impureEnvVars = fetchers.proxyImpureEnvVars ++ [ "GOPROXY" ];
+    };
+
+  mkVendorEnv =
+    { go
+    , modulesStruct
+    , localReplaceCommands ? [ ]
+    , defaultPackage ? ""
+    , goMod
+    , pwd
+    }:
+    let
+      localReplaceCommands =
+        let
+          localReplaceAttrs = filterAttrs (n: v: hasAttr "path" v) goMod.replace;
+          commands = (
+            mapAttrsToList
+              (name: value: (
+                ''
+                  mkdir -p $(dirname vendor/${name})
+                  ln -s ${pwd + "/${value.path}"} vendor/${name}
+                ''
+              ))
+              localReplaceAttrs);
+        in
+        if goMod != null then commands else [ ];
+
+      sources = mapAttrs
+        (goPackagePath: meta: fetchGoModule {
+          goPackagePath = meta.replaced or goPackagePath;
+          inherit (meta) version hash;
+          inherit go;
+        })
+        modulesStruct.mod;
+    in
+    runCommand "vendor-env"
+      {
+        nativeBuildInputs = [ go ];
+        json = toJSON (filterAttrs (n: _: n != defaultPackage) modulesStruct.mod);
+
+        sources = toJSON (filterAttrs (n: _: n != defaultPackage) sources);
+
+        passthru = {
+          inherit sources;
+        };
+
+        passAsFile = [ "json" "sources" ];
+      }
+      (
+        ''
+          mkdir vendor
+
+          export GOCACHE=$TMPDIR/go-cache
+          export GOPATH="$TMPDIR/go"
+
+          ${internal.symlink}
+          ${concatStringsSep "\n" localReplaceCommands}
+
+          mv vendor $out
+        ''
+      );
+
+  # Select Go attribute based on version specified in go.mod
+  selectGo = attrs: goMod: attrs.go or (if goMod == null then pkgs.go else
+  (
+    let
+      goVersion = goMod.go;
+      goAttr = "go_" + (replaceStrings [ "." ] [ "_" ] goVersion);
+    in
+    (
+      if hasAttr goAttr pkgs then pkgs.${goAttr}
+      else trace "go.mod specified Go version ${goVersion} but doesn't exist. Falling back to ${pkgs.go.version}." pkgs.go
+    )
+  ));
+
+  # Strip the rubbish that Go adds to versions, and fall back to a version based on the date if it's a placeholder value
+  stripVersion = version:
+    let
+      parts = elemAt (split "(\\+|-)" (removePrefix "v" version));
+      v = parts 0;
+      d = parts 2;
+    in
+    if v != "0.0.0" then v else "unstable-" + (concatStringsSep "-" [
+      (substring 0 4 d)
+      (substring 4 2 d)
+      (substring 6 2 d)
+    ]);
+
+  mkGoEnv =
+    { pwd
+    }@attrs:
+    let
+      goMod = parseGoMod (readFile "${toString pwd}/go.mod");
+      modulesStruct = fromTOML (readFile "${toString pwd}/gomod2nix.toml");
+
+      go = selectGo attrs goMod;
+
+      vendorEnv = mkVendorEnv {
+        inherit go modulesStruct pwd goMod;
+      };
+
+    in
+    stdenv.mkDerivation (removeAttrs attrs [ "pwd" ] // {
+      name = "${baseNameOf goMod.module}-env";
+
+      dontUnpack = true;
+      dontConfigure = true;
+      dontInstall = true;
+
+      propagatedNativeBuildInputs = [ go ];
+
+      GO_NO_VENDOR_CHECKS = "1";
+
+      GO111MODULE = "on";
+      GOFLAGS = "-mod=vendor";
+
+      preferLocalBuild = true;
+
+      buildPhase = ''
+        mkdir $out
+
+        export GOCACHE=$TMPDIR/go-cache
+        export GOPATH="$out"
+        export GOSUMDB=off
+        export GOPROXY=off
+
+      '' + optionalString (pathExists (pwd + "/tools.go")) ''
+        mkdir source
+        cp ${pwd + "/go.mod"} source/go.mod
+        cp ${pwd + "/go.sum"} source/go.sum
+        cp ${pwd + "/tools.go"} source/tools.go
+        cd source
+        ln -s ${vendorEnv} vendor
+
+        ${internal.install}
+      '';
+    });
+
+  buildGoApplication =
+    { modules ? pwd + "/gomod2nix.toml"
+    , src ? pwd
+    , pwd ? null
+    , nativeBuildInputs ? [ ]
+    , allowGoReference ? false
+    , meta ? { }
+    , passthru ? { }
+    , tags ? [ ]
+
+      # needed for buildFlags{,Array} warning
+    , buildFlags ? ""
+    , buildFlagsArray ? ""
+
+    , ...
+    }@attrs:
+    let
+      modulesStruct = fromTOML (readFile modules);
+
+      goModPath = "${toString pwd}/go.mod";
+
+      goMod =
+        if pwd != null && pathExists goModPath
+        then parseGoMod (readFile goModPath)
+        else null;
+
+      go = selectGo attrs goMod;
+
+      removeReferences = [ ] ++ optional (!allowGoReference) go;
+
+      defaultPackage = modulesStruct.goPackagePath or "";
+
+      vendorEnv = mkVendorEnv {
+        inherit go modulesStruct defaultPackage goMod pwd;
+      };
+
+    in
+    warnIf (buildFlags != "" || buildFlagsArray != "")
+      "Use the `ldflags` and/or `tags` attributes instead of `buildFlags`/`buildFlagsArray`"
+      stdenv.mkDerivation
+      (optionalAttrs (defaultPackage != "")
+        {
+          pname = attrs.pname or baseNameOf defaultPackage;
+          version = stripVersion (modulesStruct.mod.${defaultPackage}).version;
+          src = vendorEnv.passthru.sources.${defaultPackage};
+        } // optionalAttrs (hasAttr "subPackages" modulesStruct) {
+        subPackages = modulesStruct.subPackages;
+      } // attrs // {
+        nativeBuildInputs = [ removeReferencesTo go ] ++ nativeBuildInputs;
+
+        inherit (go) GOOS GOARCH;
+
+        GO_NO_VENDOR_CHECKS = "1";
+
+        GO111MODULE = "on";
+        GOFLAGS = "-mod=vendor";
+
+        configurePhase = attrs.configurePhase or ''
+          runHook preConfigure
+
+          export GOCACHE=$TMPDIR/go-cache
+          export GOPATH="$TMPDIR/go"
+          export GOSUMDB=off
+          export GOPROXY=off
+          cd "$modRoot"
+          if [ -n "${vendorEnv}" ]; then
+              rm -rf vendor
+              ln -s ${vendorEnv} vendor
+          fi
+
+          runHook postConfigure
+        '';
+
+        buildPhase = attrs.buildPhase or ''
+          runHook preBuild
+
+          exclude='\(/_\|examples\|Godeps\|testdata'
+          if [[ -n "$excludedPackages" ]]; then
+            IFS=' ' read -r -a excludedArr <<<$excludedPackages
+            printf -v excludedAlternates '%s\\|' "''${excludedArr[@]}"
+            excludedAlternates=''${excludedAlternates%\\|} # drop final \| added by printf
+            exclude+='\|'"$excludedAlternates"
+          fi
+          exclude+='\)'
+
+          buildGoDir() {
+            local d; local cmd;
+            cmd="$1"
+            d="$2"
+            . $TMPDIR/buildFlagsArray
+            local OUT
+            if ! OUT="$(go $cmd $buildFlags "''${buildFlagsArray[@]}" ''${tags:+-tags=${concatStringsSep "," tags}} ''${ldflags:+-ldflags="$ldflags"} -v -p $NIX_BUILD_CORES $d 2>&1)"; then
+              if echo "$OUT" | grep -qE 'imports .*?: no Go files in'; then
+                echo "$OUT" >&2
+                return 1
+              fi
+              if ! echo "$OUT" | grep -qE '(no( buildable| non-test)?|build constraints exclude all) Go (source )?files'; then
+                echo "$OUT" >&2
+                return 1
+              fi
+            fi
+            if [ -n "$OUT" ]; then
+              echo "$OUT" >&2
+            fi
+            return 0
+          }
+
+          getGoDirs() {
+            local type;
+            type="$1"
+            if [ -n "$subPackages" ]; then
+              echo "$subPackages" | sed "s,\(^\| \),\1./,g"
+            else
+              find . -type f -name \*$type.go -exec dirname {} \; | grep -v "/vendor/" | sort --unique | grep -v "$exclude"
+            fi
+          }
+
+          if (( "''${NIX_DEBUG:-0}" >= 1 )); then
+            buildFlagsArray+=(-x)
+          fi
+
+          if [ ''${#buildFlagsArray[@]} -ne 0 ]; then
+            declare -p buildFlagsArray > $TMPDIR/buildFlagsArray
+          else
+            touch $TMPDIR/buildFlagsArray
+          fi
+          if [ -z "$enableParallelBuilding" ]; then
+              export NIX_BUILD_CORES=1
+          fi
+          for pkg in $(getGoDirs ""); do
+            echo "Building subPackage $pkg"
+            buildGoDir install "$pkg"
+          done
+        '' + optionalString (stdenv.hostPlatform != stdenv.buildPlatform) ''
+          # normalize cross-compiled builds w.r.t. native builds
+          (
+            dir=$GOPATH/bin/${go.GOOS}_${go.GOARCH}
+            if [[ -n "$(shopt -s nullglob; echo $dir/*)" ]]; then
+              mv $dir/* $dir/..
+            fi
+            if [[ -d $dir ]]; then
+              rmdir $dir
+            fi
+          )
+        '' + ''
+          runHook postBuild
+        '';
+
+        doCheck = attrs.doCheck or true;
+        checkPhase = attrs.checkPhase or ''
+          runHook preCheck
+
+          for pkg in $(getGoDirs test); do
+            buildGoDir test $checkFlags "$pkg"
+          done
+
+          runHook postCheck
+        '';
+
+        installPhase = attrs.installPhase or ''
+          runHook preInstall
+
+          mkdir -p $out
+          dir="$GOPATH/bin"
+          [ -e "$dir" ] && cp -r $dir $out
+
+          runHook postInstall
+        '';
+
+        preFixup = (attrs.preFixup or "") + ''
+          find $out/{bin,libexec,lib} -type f 2>/dev/null | xargs -r ${removeExpr removeReferences} || true
+        '';
+
+        strictDeps = true;
+
+        disallowedReferences = optional (!allowGoReference) go;
+
+        passthru = { inherit go vendorEnv; } // passthru;
+
+        meta = { platforms = go.meta.platforms or platforms.all; } // meta;
+      });
+
+in
+{
+  inherit buildGoApplication mkGoEnv;
+}
diff --git a/nix/builder/fetch.sh b/nix/builder/fetch.sh
new file mode 100644
index 0000000..8c6bdb3
--- /dev/null
+++ b/nix/builder/fetch.sh
@@ -0,0 +1,13 @@
+source $stdenv/setup
+
+export HOME=$(mktemp -d)
+
+# Call once first outside of subshell for better error reporting
+go mod download "$goPackagePath@$version"
+
+dir=$(go mod download --json "$goPackagePath@$version" | jq -r .Dir)
+
+chmod -R +w $dir
+find $dir -iname ".ds_store" | xargs -r rm -rf
+
+cp -r $dir $out
diff --git a/nix/builder/install/install.go b/nix/builder/install/install.go
new file mode 100644
index 0000000..4f770b0
--- /dev/null
+++ b/nix/builder/install/install.go
@@ -0,0 +1,57 @@
+package main
+
+import (
+	"fmt"
+	"go/parser"
+	"go/token"
+	"io"
+	"os"
+	"os/exec"
+	"strconv"
+)
+
+const filename = "tools.go"
+
+func main() {
+	fset := token.NewFileSet()
+
+	var src []byte
+	{
+		f, err := os.Open(filename)
+		if err != nil {
+			panic(err)
+		}
+
+		src, err = io.ReadAll(f)
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	f, err := parser.ParseFile(fset, filename, src, parser.ImportsOnly)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	for _, s := range f.Imports {
+		path, err := strconv.Unquote(s.Path.Value)
+		if err != nil {
+			panic(err)
+		}
+
+		cmd := exec.Command("go", "install", path)
+
+		fmt.Printf("Executing '%s'\n", cmd)
+
+		err = cmd.Start()
+		if err != nil {
+			panic(err)
+		}
+
+		err = cmd.Wait()
+		if err != nil {
+			panic(err)
+		}
+	}
+}
diff --git a/nix/builder/parser.nix b/nix/builder/parser.nix
new file mode 100644
index 0000000..eb6f75e
--- /dev/null
+++ b/nix/builder/parser.nix
@@ -0,0 +1,141 @@
+# Parse go.mod in Nix
+# Returns a Nix structure with the contents of the go.mod passed in
+# in normalised form.
+
+let
+  inherit (builtins) elemAt mapAttrs split foldl' match filter typeOf hasAttr length;
+
+  # Strip lines with comments & other junk
+  stripStr = s: elemAt (split "^ *" (elemAt (split " *$" s) 0)) 2;
+  stripLines = initialLines: foldl' (acc: f: f acc) initialLines [
+    # Strip comments
+    (lines: map
+      (l: stripStr (elemAt (splitString "//" l) 0))
+      lines)
+
+    # Strip leading tabs characters
+    (lines: map (l: elemAt (match "(\t)?(.*)" l) 1) lines)
+
+    # Filter empty lines
+    (filter (l: l != ""))
+  ];
+
+  # Parse lines into a structure
+  parseLines = lines: (foldl'
+    (acc: l:
+      let
+        m = match "([^ )]*) *(.*)" l;
+        directive = elemAt m 0;
+        rest = elemAt m 1;
+
+        # Maintain parser state (inside parens or not)
+        inDirective =
+          if rest == "(" then directive
+          else if rest == ")" then null
+          else acc.inDirective
+        ;
+
+      in
+      {
+        data = (acc.data // (
+          if directive == "" && rest == ")" then { }
+          else if inDirective != null && rest == "(" && ! hasAttr inDirective acc.data then {
+            ${inDirective} = { };
+          }
+          else if rest == "(" || rest == ")" then { }
+          else if inDirective != null then {
+            ${inDirective} = acc.data.${inDirective} // { ${directive} = rest; };
+          } else if directive == "replace" then
+            (
+              let
+                segments = split " => " rest;
+                getSegment = elemAt segments;
+              in
+              assert length segments == 3; {
+                replace = acc.data.replace // {
+                  ${getSegment 0} = "=> ${getSegment 2}";
+                };
+              }
+            )
+          else {
+            ${directive} = rest;
+          }
+        )
+        );
+        inherit inDirective;
+      })
+    {
+      inDirective = null;
+      data = {
+        require = { };
+        replace = { };
+        exclude = { };
+      };
+    }
+    lines
+  ).data;
+
+  normaliseDirectives = data: (
+    let
+      normaliseString = s:
+        let
+          m = builtins.match "([^ ]+) (.+)" s;
+        in
+        {
+          ${elemAt m 0} = elemAt m 1;
+        };
+      require = data.require or { };
+      replace = data.replace or { };
+      exclude = data.exclude or { };
+    in
+    data // {
+      require =
+        if typeOf require == "string" then normaliseString require
+        else require;
+      replace =
+        if typeOf replace == "string" then normaliseString replace
+        else replace;
+    }
+  );
+
+  parseVersion = ver:
+    let
+      m = elemAt (match "([^-]+)-?([^-]*)-?([^-]*)" ver);
+      v = elemAt (match "([^+]+)\\+?(.*)" (m 0));
+    in
+    {
+      version = v 0;
+      versionSuffix = v 1;
+      date = m 1;
+      rev = m 2;
+    };
+
+  parseReplace = data: (
+    data // {
+      replace =
+        mapAttrs
+          (_: v:
+            let
+              m = match "=> ([^ ]+) (.+)" v;
+              m2 = match "=> (.*+)" v;
+            in
+            if m != null then {
+              goPackagePath = elemAt m 0;
+              version = elemAt m 1;
+            } else {
+              path = elemAt m2 0;
+            })
+          data.replace;
+    }
+  );
+
+  splitString = sep: s: filter (t: t != [ ]) (split sep s);
+
+in
+contents:
+foldl' (acc: f: f acc) (splitString "\n" contents) [
+  stripLines
+  parseLines
+  normaliseDirectives
+  parseReplace
+]
diff --git a/nix/builder/symlink/symlink.go b/nix/builder/symlink/symlink.go
new file mode 100644
index 0000000..3dbb383
--- /dev/null
+++ b/nix/builder/symlink/symlink.go
@@ -0,0 +1,110 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+)
+
+type Package struct {
+	GoPackagePath string `json:"-"`
+	Version       string `json:"version"`
+	Hash          string `json:"hash"`
+	ReplacedPath  string `json:"replaced,omitempty"`
+}
+
+// type Output struct {
+// 	SchemaVersion int                 `json:"schema"`
+// 	Mod           map[string]*Package `json:"mod"`
+// }
+
+func main() {
+
+	// var output Output
+	sources := make(map[string]string)
+	pkgs := make(map[string]*Package)
+
+	{
+		b, err := ioutil.ReadFile(os.Getenv("sourcesPath"))
+		if err != nil {
+			panic(err)
+		}
+
+		err = json.Unmarshal(b, &sources)
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	{
+		b, err := ioutil.ReadFile(os.Getenv("jsonPath"))
+		if err != nil {
+			panic(err)
+		}
+
+		err = json.Unmarshal(b, &pkgs)
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	keys := make([]string, 0, len(pkgs))
+	for key := range pkgs {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+
+	// Iterate, in reverse order
+	for i := len(keys) - 1; i >= 0; i-- {
+		key := keys[i]
+		src := sources[key]
+
+		paths := []string{key}
+
+		for _, path := range paths {
+
+			vendorDir := filepath.Join("vendor", filepath.Dir(path))
+			if err := os.MkdirAll(vendorDir, 0755); err != nil {
+				panic(err)
+			}
+
+			if _, err := os.Stat(filepath.Join("vendor", path)); err == nil {
+				files, err := ioutil.ReadDir(src)
+				if err != nil {
+					panic(err)
+				}
+
+				for _, f := range files {
+					innerSrc := filepath.Join(src, f.Name())
+					dst := filepath.Join("vendor", path, f.Name())
+					if err := os.Symlink(innerSrc, dst); err != nil {
+						// assume it's an existing directory, try to link the directory content instead.
+						// TODO should we do this recursively
+						files, err := ioutil.ReadDir(innerSrc)
+						if err != nil {
+							panic(err)
+						}
+						for _, f := range files {
+							if err := os.Symlink(filepath.Join(innerSrc, f.Name()), filepath.Join(dst, f.Name())); err != nil {
+								fmt.Println("ignore symlink error", filepath.Join(innerSrc, f.Name()), filepath.Join(dst, f.Name()))
+							}
+						}
+					}
+				}
+
+				continue
+			}
+
+			// If the file doesn't already exist, just create a simple symlink
+			err := os.Symlink(src, filepath.Join("vendor", path))
+			if err != nil {
+				panic(err)
+			}
+
+		}
+	}
+
+}