aboutsummaryrefslogtreecommitdiff
path: root/nix/builder
diff options
context:
space:
mode:
Diffstat (limited to 'nix/builder')
-rw-r--r--nix/builder/default.nix387
-rw-r--r--nix/builder/fetch.sh13
-rw-r--r--nix/builder/install/install.go57
-rw-r--r--nix/builder/parser.nix141
-rw-r--r--nix/builder/symlink/symlink.go110
5 files changed, 708 insertions, 0 deletions
diff --git a/nix/builder/default.nix b/nix/builder/default.nix
new file mode 100644
index 0000000..8b8ae19
--- /dev/null
+++ b/nix/builder/default.nix
@@ -0,0 +1,387 @@
+{ stdenv
+, stdenvNoCC
+, runCommand
+, buildEnv
+, lib
+, fetchgit
+, removeReferencesTo
+, jq
+, cacert
+, pkgs
+, pkgsBuildBuild
+}:
+let
+
+ inherit (builtins) substring toJSON hasAttr trace split readFile elemAt;
+ inherit (lib)
+ concatStringsSep replaceStrings removePrefix optionalString pathExists
+ optional concatMapStrings fetchers filterAttrs mapAttrs mapAttrsToList
+ warnIf optionalAttrs platforms
+ ;
+
+ parseGoMod = import ./parser.nix;
+
+ removeExpr = refs: ''remove-references-to ${concatMapStrings (ref: " -t ${ref}") refs}'';
+
+ # Internal only build-time attributes
+ internal =
+ let
+ mkInternalPkg = name: src: pkgsBuildBuild.runCommand "gomod2nix-${name}"
+ {
+ inherit (pkgsBuildBuild.go) GOOS GOARCH;
+ nativeBuildInputs = [ pkgsBuildBuild.go ];
+ } ''
+ export HOME=$(mktemp -d)
+ cp ${src} src.go
+ go build -o $out src.go
+ '';
+ in
+ {
+
+ # Create a symlink tree of vendored sources
+ symlink = mkInternalPkg "symlink" ./symlink/symlink.go;
+
+ # Install development dependencies from tools.go
+ install = mkInternalPkg "symlink" ./install/install.go;
+
+ };
+
+ fetchGoModule =
+ { hash
+ , goPackagePath
+ , version
+ , go ? pkgs.go
+ }:
+ stdenvNoCC.mkDerivation {
+ name = "${baseNameOf goPackagePath}_${version}";
+ builder = ./fetch.sh;
+ inherit goPackagePath version;
+ nativeBuildInputs = [ go jq ];
+ outputHashMode = "recursive";
+ outputHashAlgo = null;
+ outputHash = hash;
+ SSL_CERT_FILE = "${cacert}/etc/ssl/certs/ca-bundle.crt";
+ impureEnvVars = fetchers.proxyImpureEnvVars ++ [ "GOPROXY" ];
+ };
+
+ mkVendorEnv =
+ { go
+ , modulesStruct
+ , localReplaceCommands ? [ ]
+ , defaultPackage ? ""
+ , goMod
+ , pwd
+ }:
+ let
+ localReplaceCommands =
+ let
+ localReplaceAttrs = filterAttrs (n: v: hasAttr "path" v) goMod.replace;
+ commands = (
+ mapAttrsToList
+ (name: value: (
+ ''
+ mkdir -p $(dirname vendor/${name})
+ ln -s ${pwd + "/${value.path}"} vendor/${name}
+ ''
+ ))
+ localReplaceAttrs);
+ in
+ if goMod != null then commands else [ ];
+
+ sources = mapAttrs
+ (goPackagePath: meta: fetchGoModule {
+ goPackagePath = meta.replaced or goPackagePath;
+ inherit (meta) version hash;
+ inherit go;
+ })
+ modulesStruct.mod;
+ in
+ runCommand "vendor-env"
+ {
+ nativeBuildInputs = [ go ];
+ json = toJSON (filterAttrs (n: _: n != defaultPackage) modulesStruct.mod);
+
+ sources = toJSON (filterAttrs (n: _: n != defaultPackage) sources);
+
+ passthru = {
+ inherit sources;
+ };
+
+ passAsFile = [ "json" "sources" ];
+ }
+ (
+ ''
+ mkdir vendor
+
+ export GOCACHE=$TMPDIR/go-cache
+ export GOPATH="$TMPDIR/go"
+
+ ${internal.symlink}
+ ${concatStringsSep "\n" localReplaceCommands}
+
+ mv vendor $out
+ ''
+ );
+
+ # Select Go attribute based on version specified in go.mod
+ selectGo = attrs: goMod: attrs.go or (if goMod == null then pkgs.go else
+ (
+ let
+ goVersion = goMod.go;
+ goAttr = "go_" + (replaceStrings [ "." ] [ "_" ] goVersion);
+ in
+ (
+ if hasAttr goAttr pkgs then pkgs.${goAttr}
+ else trace "go.mod specified Go version ${goVersion} but doesn't exist. Falling back to ${pkgs.go.version}." pkgs.go
+ )
+ ));
+
+ # Strip the rubbish that Go adds to versions, and fall back to a version based on the date if it's a placeholder value
+ stripVersion = version:
+ let
+ parts = elemAt (split "(\\+|-)" (removePrefix "v" version));
+ v = parts 0;
+ d = parts 2;
+ in
+ if v != "0.0.0" then v else "unstable-" + (concatStringsSep "-" [
+ (substring 0 4 d)
+ (substring 4 2 d)
+ (substring 6 2 d)
+ ]);
+
+ mkGoEnv =
+ { pwd
+ }@attrs:
+ let
+ goMod = parseGoMod (readFile "${toString pwd}/go.mod");
+ modulesStruct = fromTOML (readFile "${toString pwd}/gomod2nix.toml");
+
+ go = selectGo attrs goMod;
+
+ vendorEnv = mkVendorEnv {
+ inherit go modulesStruct pwd goMod;
+ };
+
+ in
+ stdenv.mkDerivation (removeAttrs attrs [ "pwd" ] // {
+ name = "${baseNameOf goMod.module}-env";
+
+ dontUnpack = true;
+ dontConfigure = true;
+ dontInstall = true;
+
+ propagatedNativeBuildInputs = [ go ];
+
+ GO_NO_VENDOR_CHECKS = "1";
+
+ GO111MODULE = "on";
+ GOFLAGS = "-mod=vendor";
+
+ preferLocalBuild = true;
+
+ buildPhase = ''
+ mkdir $out
+
+ export GOCACHE=$TMPDIR/go-cache
+ export GOPATH="$out"
+ export GOSUMDB=off
+ export GOPROXY=off
+
+ '' + optionalString (pathExists (pwd + "/tools.go")) ''
+ mkdir source
+ cp ${pwd + "/go.mod"} source/go.mod
+ cp ${pwd + "/go.sum"} source/go.sum
+ cp ${pwd + "/tools.go"} source/tools.go
+ cd source
+ ln -s ${vendorEnv} vendor
+
+ ${internal.install}
+ '';
+ });
+
+ buildGoApplication =
+ { modules ? pwd + "/gomod2nix.toml"
+ , src ? pwd
+ , pwd ? null
+ , nativeBuildInputs ? [ ]
+ , allowGoReference ? false
+ , meta ? { }
+ , passthru ? { }
+ , tags ? [ ]
+
+ # needed for buildFlags{,Array} warning
+ , buildFlags ? ""
+ , buildFlagsArray ? ""
+
+ , ...
+ }@attrs:
+ let
+ modulesStruct = fromTOML (readFile modules);
+
+ goModPath = "${toString pwd}/go.mod";
+
+ goMod =
+ if pwd != null && pathExists goModPath
+ then parseGoMod (readFile goModPath)
+ else null;
+
+ go = selectGo attrs goMod;
+
+ removeReferences = [ ] ++ optional (!allowGoReference) go;
+
+ defaultPackage = modulesStruct.goPackagePath or "";
+
+ vendorEnv = mkVendorEnv {
+ inherit go modulesStruct defaultPackage goMod pwd;
+ };
+
+ in
+ warnIf (buildFlags != "" || buildFlagsArray != "")
+ "Use the `ldflags` and/or `tags` attributes instead of `buildFlags`/`buildFlagsArray`"
+ stdenv.mkDerivation
+ (optionalAttrs (defaultPackage != "")
+ {
+ pname = attrs.pname or baseNameOf defaultPackage;
+ version = stripVersion (modulesStruct.mod.${defaultPackage}).version;
+ src = vendorEnv.passthru.sources.${defaultPackage};
+ } // optionalAttrs (hasAttr "subPackages" modulesStruct) {
+ subPackages = modulesStruct.subPackages;
+ } // attrs // {
+ nativeBuildInputs = [ removeReferencesTo go ] ++ nativeBuildInputs;
+
+ inherit (go) GOOS GOARCH;
+
+ GO_NO_VENDOR_CHECKS = "1";
+
+ GO111MODULE = "on";
+ GOFLAGS = "-mod=vendor";
+
+ configurePhase = attrs.configurePhase or ''
+ runHook preConfigure
+
+ export GOCACHE=$TMPDIR/go-cache
+ export GOPATH="$TMPDIR/go"
+ export GOSUMDB=off
+ export GOPROXY=off
+ cd "$modRoot"
+ if [ -n "${vendorEnv}" ]; then
+ rm -rf vendor
+ ln -s ${vendorEnv} vendor
+ fi
+
+ runHook postConfigure
+ '';
+
+ buildPhase = attrs.buildPhase or ''
+ runHook preBuild
+
+ exclude='\(/_\|examples\|Godeps\|testdata'
+ if [[ -n "$excludedPackages" ]]; then
+ IFS=' ' read -r -a excludedArr <<<$excludedPackages
+ printf -v excludedAlternates '%s\\|' "''${excludedArr[@]}"
+ excludedAlternates=''${excludedAlternates%\\|} # drop final \| added by printf
+ exclude+='\|'"$excludedAlternates"
+ fi
+ exclude+='\)'
+
+ buildGoDir() {
+ local d; local cmd;
+ cmd="$1"
+ d="$2"
+ . $TMPDIR/buildFlagsArray
+ local OUT
+ if ! OUT="$(go $cmd $buildFlags "''${buildFlagsArray[@]}" ''${tags:+-tags=${concatStringsSep "," tags}} ''${ldflags:+-ldflags="$ldflags"} -v -p $NIX_BUILD_CORES $d 2>&1)"; then
+ if echo "$OUT" | grep -qE 'imports .*?: no Go files in'; then
+ echo "$OUT" >&2
+ return 1
+ fi
+ if ! echo "$OUT" | grep -qE '(no( buildable| non-test)?|build constraints exclude all) Go (source )?files'; then
+ echo "$OUT" >&2
+ return 1
+ fi
+ fi
+ if [ -n "$OUT" ]; then
+ echo "$OUT" >&2
+ fi
+ return 0
+ }
+
+ getGoDirs() {
+ local type;
+ type="$1"
+ if [ -n "$subPackages" ]; then
+ echo "$subPackages" | sed "s,\(^\| \),\1./,g"
+ else
+ find . -type f -name \*$type.go -exec dirname {} \; | grep -v "/vendor/" | sort --unique | grep -v "$exclude"
+ fi
+ }
+
+ if (( "''${NIX_DEBUG:-0}" >= 1 )); then
+ buildFlagsArray+=(-x)
+ fi
+
+ if [ ''${#buildFlagsArray[@]} -ne 0 ]; then
+ declare -p buildFlagsArray > $TMPDIR/buildFlagsArray
+ else
+ touch $TMPDIR/buildFlagsArray
+ fi
+ if [ -z "$enableParallelBuilding" ]; then
+ export NIX_BUILD_CORES=1
+ fi
+ for pkg in $(getGoDirs ""); do
+ echo "Building subPackage $pkg"
+ buildGoDir install "$pkg"
+ done
+ '' + optionalString (stdenv.hostPlatform != stdenv.buildPlatform) ''
+ # normalize cross-compiled builds w.r.t. native builds
+ (
+ dir=$GOPATH/bin/${go.GOOS}_${go.GOARCH}
+ if [[ -n "$(shopt -s nullglob; echo $dir/*)" ]]; then
+ mv $dir/* $dir/..
+ fi
+ if [[ -d $dir ]]; then
+ rmdir $dir
+ fi
+ )
+ '' + ''
+ runHook postBuild
+ '';
+
+ doCheck = attrs.doCheck or true;
+ checkPhase = attrs.checkPhase or ''
+ runHook preCheck
+
+ for pkg in $(getGoDirs test); do
+ buildGoDir test $checkFlags "$pkg"
+ done
+
+ runHook postCheck
+ '';
+
+ installPhase = attrs.installPhase or ''
+ runHook preInstall
+
+ mkdir -p $out
+ dir="$GOPATH/bin"
+ [ -e "$dir" ] && cp -r $dir $out
+
+ runHook postInstall
+ '';
+
+ preFixup = (attrs.preFixup or "") + ''
+ find $out/{bin,libexec,lib} -type f 2>/dev/null | xargs -r ${removeExpr removeReferences} || true
+ '';
+
+ strictDeps = true;
+
+ disallowedReferences = optional (!allowGoReference) go;
+
+ passthru = { inherit go vendorEnv; } // passthru;
+
+ meta = { platforms = go.meta.platforms or platforms.all; } // meta;
+ });
+
+in
+{
+ inherit buildGoApplication mkGoEnv;
+}
diff --git a/nix/builder/fetch.sh b/nix/builder/fetch.sh
new file mode 100644
index 0000000..8c6bdb3
--- /dev/null
+++ b/nix/builder/fetch.sh
@@ -0,0 +1,13 @@
+source $stdenv/setup
+
+export HOME=$(mktemp -d)
+
+# Call once first outside of subshell for better error reporting
+go mod download "$goPackagePath@$version"
+
+dir=$(go mod download --json "$goPackagePath@$version" | jq -r .Dir)
+
+chmod -R +w $dir
+find $dir -iname ".ds_store" | xargs -r rm -rf
+
+cp -r $dir $out
diff --git a/nix/builder/install/install.go b/nix/builder/install/install.go
new file mode 100644
index 0000000..4f770b0
--- /dev/null
+++ b/nix/builder/install/install.go
@@ -0,0 +1,57 @@
+package main
+
+import (
+ "fmt"
+ "go/parser"
+ "go/token"
+ "io"
+ "os"
+ "os/exec"
+ "strconv"
+)
+
+const filename = "tools.go"
+
+func main() {
+ fset := token.NewFileSet()
+
+ var src []byte
+ {
+ f, err := os.Open(filename)
+ if err != nil {
+ panic(err)
+ }
+
+ src, err = io.ReadAll(f)
+ if err != nil {
+ panic(err)
+ }
+ }
+
+ f, err := parser.ParseFile(fset, filename, src, parser.ImportsOnly)
+ if err != nil {
+ fmt.Println(err)
+ return
+ }
+
+ for _, s := range f.Imports {
+ path, err := strconv.Unquote(s.Path.Value)
+ if err != nil {
+ panic(err)
+ }
+
+ cmd := exec.Command("go", "install", path)
+
+ fmt.Printf("Executing '%s'\n", cmd)
+
+ err = cmd.Start()
+ if err != nil {
+ panic(err)
+ }
+
+ err = cmd.Wait()
+ if err != nil {
+ panic(err)
+ }
+ }
+}
diff --git a/nix/builder/parser.nix b/nix/builder/parser.nix
new file mode 100644
index 0000000..eb6f75e
--- /dev/null
+++ b/nix/builder/parser.nix
@@ -0,0 +1,141 @@
+# Parse go.mod in Nix
+# Returns a Nix structure with the contents of the go.mod passed in
+# in normalised form.
+
+let
+ inherit (builtins) elemAt mapAttrs split foldl' match filter typeOf hasAttr length;
+
+ # Strip lines with comments & other junk
+ stripStr = s: elemAt (split "^ *" (elemAt (split " *$" s) 0)) 2;
+ stripLines = initialLines: foldl' (acc: f: f acc) initialLines [
+ # Strip comments
+ (lines: map
+ (l: stripStr (elemAt (splitString "//" l) 0))
+ lines)
+
+ # Strip leading tabs characters
+ (lines: map (l: elemAt (match "(\t)?(.*)" l) 1) lines)
+
+ # Filter empty lines
+ (filter (l: l != ""))
+ ];
+
+ # Parse lines into a structure
+ parseLines = lines: (foldl'
+ (acc: l:
+ let
+ m = match "([^ )]*) *(.*)" l;
+ directive = elemAt m 0;
+ rest = elemAt m 1;
+
+ # Maintain parser state (inside parens or not)
+ inDirective =
+ if rest == "(" then directive
+ else if rest == ")" then null
+ else acc.inDirective
+ ;
+
+ in
+ {
+ data = (acc.data // (
+ if directive == "" && rest == ")" then { }
+ else if inDirective != null && rest == "(" && ! hasAttr inDirective acc.data then {
+ ${inDirective} = { };
+ }
+ else if rest == "(" || rest == ")" then { }
+ else if inDirective != null then {
+ ${inDirective} = acc.data.${inDirective} // { ${directive} = rest; };
+ } else if directive == "replace" then
+ (
+ let
+ segments = split " => " rest;
+ getSegment = elemAt segments;
+ in
+ assert length segments == 3; {
+ replace = acc.data.replace // {
+ ${getSegment 0} = "=> ${getSegment 2}";
+ };
+ }
+ )
+ else {
+ ${directive} = rest;
+ }
+ )
+ );
+ inherit inDirective;
+ })
+ {
+ inDirective = null;
+ data = {
+ require = { };
+ replace = { };
+ exclude = { };
+ };
+ }
+ lines
+ ).data;
+
+ normaliseDirectives = data: (
+ let
+ normaliseString = s:
+ let
+ m = builtins.match "([^ ]+) (.+)" s;
+ in
+ {
+ ${elemAt m 0} = elemAt m 1;
+ };
+ require = data.require or { };
+ replace = data.replace or { };
+ exclude = data.exclude or { };
+ in
+ data // {
+ require =
+ if typeOf require == "string" then normaliseString require
+ else require;
+ replace =
+ if typeOf replace == "string" then normaliseString replace
+ else replace;
+ }
+ );
+
+ parseVersion = ver:
+ let
+ m = elemAt (match "([^-]+)-?([^-]*)-?([^-]*)" ver);
+ v = elemAt (match "([^+]+)\\+?(.*)" (m 0));
+ in
+ {
+ version = v 0;
+ versionSuffix = v 1;
+ date = m 1;
+ rev = m 2;
+ };
+
+ parseReplace = data: (
+ data // {
+ replace =
+ mapAttrs
+ (_: v:
+ let
+ m = match "=> ([^ ]+) (.+)" v;
+ m2 = match "=> (.*+)" v;
+ in
+ if m != null then {
+ goPackagePath = elemAt m 0;
+ version = elemAt m 1;
+ } else {
+ path = elemAt m2 0;
+ })
+ data.replace;
+ }
+ );
+
+ splitString = sep: s: filter (t: t != [ ]) (split sep s);
+
+in
+contents:
+foldl' (acc: f: f acc) (splitString "\n" contents) [
+ stripLines
+ parseLines
+ normaliseDirectives
+ parseReplace
+]
diff --git a/nix/builder/symlink/symlink.go b/nix/builder/symlink/symlink.go
new file mode 100644
index 0000000..3dbb383
--- /dev/null
+++ b/nix/builder/symlink/symlink.go
@@ -0,0 +1,110 @@
+package main
+
+import (
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "sort"
+)
+
+type Package struct {
+ GoPackagePath string `json:"-"`
+ Version string `json:"version"`
+ Hash string `json:"hash"`
+ ReplacedPath string `json:"replaced,omitempty"`
+}
+
+// type Output struct {
+// SchemaVersion int `json:"schema"`
+// Mod map[string]*Package `json:"mod"`
+// }
+
+func main() {
+
+ // var output Output
+ sources := make(map[string]string)
+ pkgs := make(map[string]*Package)
+
+ {
+ b, err := ioutil.ReadFile(os.Getenv("sourcesPath"))
+ if err != nil {
+ panic(err)
+ }
+
+ err = json.Unmarshal(b, &sources)
+ if err != nil {
+ panic(err)
+ }
+ }
+
+ {
+ b, err := ioutil.ReadFile(os.Getenv("jsonPath"))
+ if err != nil {
+ panic(err)
+ }
+
+ err = json.Unmarshal(b, &pkgs)
+ if err != nil {
+ panic(err)
+ }
+ }
+
+ keys := make([]string, 0, len(pkgs))
+ for key := range pkgs {
+ keys = append(keys, key)
+ }
+ sort.Strings(keys)
+
+ // Iterate, in reverse order
+ for i := len(keys) - 1; i >= 0; i-- {
+ key := keys[i]
+ src := sources[key]
+
+ paths := []string{key}
+
+ for _, path := range paths {
+
+ vendorDir := filepath.Join("vendor", filepath.Dir(path))
+ if err := os.MkdirAll(vendorDir, 0755); err != nil {
+ panic(err)
+ }
+
+ if _, err := os.Stat(filepath.Join("vendor", path)); err == nil {
+ files, err := ioutil.ReadDir(src)
+ if err != nil {
+ panic(err)
+ }
+
+ for _, f := range files {
+ innerSrc := filepath.Join(src, f.Name())
+ dst := filepath.Join("vendor", path, f.Name())
+ if err := os.Symlink(innerSrc, dst); err != nil {
+ // assume it's an existing directory, try to link the directory content instead.
+ // TODO should we do this recursively
+ files, err := ioutil.ReadDir(innerSrc)
+ if err != nil {
+ panic(err)
+ }
+ for _, f := range files {
+ if err := os.Symlink(filepath.Join(innerSrc, f.Name()), filepath.Join(dst, f.Name())); err != nil {
+ fmt.Println("ignore symlink error", filepath.Join(innerSrc, f.Name()), filepath.Join(dst, f.Name()))
+ }
+ }
+ }
+ }
+
+ continue
+ }
+
+ // If the file doesn't already exist, just create a simple symlink
+ err := os.Symlink(src, filepath.Join("vendor", path))
+ if err != nil {
+ panic(err)
+ }
+
+ }
+ }
+
+}
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-10 03:59:40 +0000