From c3958801656c7ce0362eeb8cb57f279d5466a2fc Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Tue, 19 Jul 2022 11:56:35 +0200 Subject: [PATCH 1/6] Compile with Nix --- default.nix | 30 +++ gomod2nix.toml | 111 ++++++++++ nix/builder/default.nix | 387 +++++++++++++++++++++++++++++++++ nix/builder/fetch.sh | 13 ++ nix/builder/install/install.go | 57 +++++ nix/builder/parser.nix | 141 ++++++++++++ nix/builder/symlink/symlink.go | 110 ++++++++++ 7 files changed, 849 insertions(+) create mode 100644 default.nix create mode 100644 gomod2nix.toml create mode 100644 nix/builder/default.nix create mode 100644 nix/builder/fetch.sh create mode 100644 nix/builder/install/install.go create mode 100644 nix/builder/parser.nix create mode 100644 nix/builder/symlink/symlink.go diff --git a/default.nix b/default.nix new file mode 100644 index 0000000..6c88213 --- /dev/null +++ b/default.nix @@ -0,0 +1,30 @@ +let + pkgsSrc = fetchTarball { + # As of 2022-07-19 + url = "https://github.com/NixOS/nixpkgs/archive/d2db10786f27619d5519b12b03fb10dc8ca95e59.tar.gz"; + sha256 = "0s9gigs3ylnq5b94rfcmxvrmmr3kzhs497gksajf638d5bv7zcl5"; + }; + pkgs = import pkgsSrc { + overlays = [ + (self: super: { + gomod = super.callPackage ./nix/builder { }; + }) + ]; + }; +in + pkgs.gomod.buildGoApplication { + pname = "bottin"; + version = "0.1.0"; + src = ./.; + modules = ./gomod2nix.toml; + + CGO_ENABLED=0; + + meta = with pkgs.lib; { + description = "Bottin is a cloud-native LDAP server backed by a Consul datastore"; + homepage = "https://git.deuxfleurs.fr/Deuxfleurs/bottin"; + license = licenses.gpl3Plus; + platforms = platforms.linux; + }; + } + diff --git a/gomod2nix.toml b/gomod2nix.toml new file mode 100644 index 0000000..690a31d --- /dev/null +++ b/gomod2nix.toml @@ -0,0 +1,111 @@ +schema = 3 + +[mod] + [mod."github.com/davecgh/go-spew"] + version = "v1.1.1" + hash = "sha256-nhzSUrE1fCkN0+RL04N4h8jWmRFPPPWbCuDc7Ss0akI=" + [mod."github.com/dustin/go-humanize"] + version = "v1.0.0" + hash = "sha256-gy4G1PnHD9iw2MitHX6y1y93qr3C9IncmXL7ttUMDs8=" + [mod."github.com/emersion/go-sasl"] + version = "v0.0.0-20191210011802-430746ea8b9b" + hash = "sha256-bADpAn0ZhlTTsEB3MsG8J31cQjTtHTzohX/wkL1aMIc=" + [mod."github.com/emersion/go-smtp"] + version = "v0.12.1" + hash = "sha256-fiss5y7chfHv80vIQ9Xwx3J+3qLMA63EOP4OG3DxAtI=" + [mod."github.com/go-asn1-ber/asn1-ber"] + version = "v1.3.1" + hash = "sha256-Alh6bUq9HoBDhY+n6W7xNBto/dUMxPGvucA6guarrjc=" + [mod."github.com/go-ldap/ldap/v3"] + version = "v3.1.6" + hash = "sha256-UPUdYKOoCQWgl2Onbq1Oql7XU4TeYQA/+j4atwhdKbE=" + [mod."github.com/google/gofuzz"] + version = "v1.0.0" + hash = "sha256-ZvgcSQt4kMwS6nvPp3jrlCHSH3bky1oBD6kytnEa5GM=" + [mod."github.com/google/uuid"] + version = "v1.1.1" + hash = "sha256-66PXC/RCPUyhS9PhkIPQFR3tbM2zZYDNPGXN7JJj3UE=" + [mod."github.com/gopherjs/gopherjs"] + version = "v0.0.0-20181017120253-0766667cb4d1" + hash = "sha256-AuXnjjoLbFZ85Oi8sldH117MBh+yCUB9HU5Y5syJ7Lg=" + [mod."github.com/gorilla/mux"] + version = "v1.7.3" + hash = "sha256-YZSIN7Ua+hPqSIrT+tiRz3aFqJ1EWHvwee+PptpHI28=" + [mod."github.com/gorilla/securecookie"] + version = "v1.1.1" + hash = "sha256-IBBYWfdOuXvQsb01DaA8tBizCfAE1J2KLXIn3W+NeJk=" + [mod."github.com/gorilla/sessions"] + version = "v1.2.0" + hash = "sha256-4V7yd/vf03CEsb3pz5dbLWwv7t9QgKkEhVXtc1/z5s8=" + [mod."github.com/jsimonetti/pwscheme"] + version = "v0.0.0-20220125093853-4d9895f5db73" + hash = "sha256-YF3RKU/4CWvLPgGzUd7Hk/2+41OUFuRWZgzQuqcsKg0=" + [mod."github.com/json-iterator/go"] + version = "v1.1.10" + hash = "sha256-jdS2C0WsgsWREBSj+YUzSqdZofMfUMecaOQ/lB9Mu6k=" + [mod."github.com/jtolds/gls"] + version = "v4.20.0+incompatible" + hash = "sha256-Zu5naRjnwOYBzRv0CYhIZTizA0AajzOg7mJrL7Bo/cw=" + [mod."github.com/klauspost/cpuid"] + version = "v1.2.3" + hash = "sha256-1IBlONMxKVgudV/mzNrFZB60z8w4xFjVbEU2DoIAoeg=" + [mod."github.com/konsorten/go-windows-terminal-sequences"] + version = "v1.0.3" + hash = "sha256-9HojTXKv7b3HiEhYaKXDxraEfvU5vrg47FbCjTRpOvs=" + [mod."github.com/minio/md5-simd"] + version = "v1.1.0" + hash = "sha256-jJbDwg7KlLB991wj1U6y+kJKOUxKVGQrDbM3nY+6qxE=" + [mod."github.com/minio/minio-go/v7"] + version = "v7.0.0" + hash = "sha256-xWAELgH6mWVGKFEe2gbzvigJDNk+ELmegJe09KvUqvY=" + [mod."github.com/minio/sha256-simd"] + version = "v0.1.1" + hash = "sha256-HpcuLTnpcyKe0ua2MN/ysK5cXdrwquDjrx4Y2dG6W2s=" + [mod."github.com/mitchellh/go-homedir"] + version = "v1.1.0" + hash = "sha256-oduBKXHAQG8X6aqLEpqZHs5DOKe84u6WkBwi4W6cv3k=" + [mod."github.com/modern-go/concurrent"] + version = "v0.0.0-20180228061459-e0a39a4cb421" + hash = "sha256-+bdeHUArnpkk4eMQIwXm9K249NkpwAjoTrXrGn/4VUE=" + [mod."github.com/modern-go/reflect2"] + version = "v0.0.0-20180701023420-4b7aa43c6742" + hash = "sha256-RyIwgrPwtd4lNjLGkBVxRvu5IdXLDqf5F69QWLm0zLw=" + [mod."github.com/nfnt/resize"] + version = "v0.0.0-20180221191011-83c6a9932646" + hash = "sha256-yvPV+HlDOyJsiwAcVHQkmtw8DHSXyw+cXHkigXm8rAA=" + [mod."github.com/pmezard/go-difflib"] + version = "v1.0.0" + hash = "sha256-/FtmHnaGjdvEIKAJtrUfEhV7EVo5A/eYrtdnUkuxLDA=" + [mod."github.com/sirupsen/logrus"] + version = "v1.6.0" + hash = "sha256-4v27X4yyl52BtZcZEnDe0tfvOaEq+TCcp7R8HBzreDM=" + [mod."github.com/smartystreets/assertions"] + version = "v0.0.0-20180927180507-b2de0cb4f26d" + hash = "sha256-PoE+VQEnzJogI/mDBJ6dTCCR217nFjHfYWXQt9Vr9MQ=" + [mod."github.com/smartystreets/goconvey"] + version = "v0.0.0-20190330032615-68dc04aab96a" + hash = "sha256-HD+AZE1agl1pVbQTFUKLKtjg3XdVSVLwRSu4u+UVV2M=" + [mod."github.com/stretchr/objx"] + version = "v0.1.0" + hash = "sha256-az0Vd4MG3JXfaYbj0Q6AOmNkrXgmXDeQm8+BBiDXmdA=" + [mod."github.com/stretchr/testify"] + version = "v1.3.0" + hash = "sha256-+mSebBNccNcxbY462iKTNTWmd5ZuUkUqFebccn3EtIA=" + [mod."golang.org/x/crypto"] + version = "v0.0.0-20200214034016-1d94cc7ab1c6" + hash = "sha256-fWTzdDxt/1E8Jx7b6tmYEVqqJs5FoVVya9aEK9gDbdY=" + [mod."golang.org/x/net"] + version = "v0.0.0-20190522155817-f3200d17e092" + hash = "sha256-KkNNFr+wx/pf7lSLN2ygwkQ9oCZQuef+hCtEjEX+gJE=" + [mod."golang.org/x/sys"] + version = "v0.0.0-20200223170610-d5e6a3e2c0ae" + hash = "sha256-IvG2XSER2dyrVfhYieEpHcp28LOz4FrjQqN0SCeFOek=" + [mod."golang.org/x/text"] + version = "v0.3.0" + hash = "sha256-0FFbaxF1ZuAQF3sCcA85e8MO6prFeHint36inija4NY=" + [mod."golang.org/x/tools"] + version = "v0.0.0-20190328211700-ab21143f2384" + hash = "sha256-OcjaTxx6C/cbnUZLN2ArTrOBlBCijWJVUPaMgK67MkY=" + [mod."gopkg.in/ini.v1"] + version = "v1.57.0" + hash = "sha256-WSjX+qHJ1Rf4FRMTs7udQwEBkIo+z8+EK3uB5CebrZ4=" diff --git a/nix/builder/default.nix b/nix/builder/default.nix new file mode 100644 index 0000000..8b8ae19 --- /dev/null +++ b/nix/builder/default.nix @@ -0,0 +1,387 @@ +{ stdenv +, stdenvNoCC +, runCommand +, buildEnv +, lib +, fetchgit +, removeReferencesTo +, jq +, cacert +, pkgs +, pkgsBuildBuild +}: +let + + inherit (builtins) substring toJSON hasAttr trace split readFile elemAt; + inherit (lib) + concatStringsSep replaceStrings removePrefix optionalString pathExists + optional concatMapStrings fetchers filterAttrs mapAttrs mapAttrsToList + warnIf optionalAttrs platforms + ; + + parseGoMod = import ./parser.nix; + + removeExpr = refs: ''remove-references-to ${concatMapStrings (ref: " -t ${ref}") refs}''; + + # Internal only build-time attributes + internal = + let + mkInternalPkg = name: src: pkgsBuildBuild.runCommand "gomod2nix-${name}" + { + inherit (pkgsBuildBuild.go) GOOS GOARCH; + nativeBuildInputs = [ pkgsBuildBuild.go ]; + } '' + export HOME=$(mktemp -d) + cp ${src} src.go + go build -o $out src.go + ''; + in + { + + # Create a symlink tree of vendored sources + symlink = mkInternalPkg "symlink" ./symlink/symlink.go; + + # Install development dependencies from tools.go + install = mkInternalPkg "symlink" ./install/install.go; + + }; + + fetchGoModule = + { hash + , goPackagePath + , version + , go ? pkgs.go + }: + stdenvNoCC.mkDerivation { + name = "${baseNameOf goPackagePath}_${version}"; + builder = ./fetch.sh; + inherit goPackagePath version; + nativeBuildInputs = [ go jq ]; + outputHashMode = "recursive"; + outputHashAlgo = null; + outputHash = hash; + SSL_CERT_FILE = "${cacert}/etc/ssl/certs/ca-bundle.crt"; + impureEnvVars = fetchers.proxyImpureEnvVars ++ [ "GOPROXY" ]; + }; + + mkVendorEnv = + { go + , modulesStruct + , localReplaceCommands ? [ ] + , defaultPackage ? "" + , goMod + , pwd + }: + let + localReplaceCommands = + let + localReplaceAttrs = filterAttrs (n: v: hasAttr "path" v) goMod.replace; + commands = ( + mapAttrsToList + (name: value: ( + '' + mkdir -p $(dirname vendor/${name}) + ln -s ${pwd + "/${value.path}"} vendor/${name} + '' + )) + localReplaceAttrs); + in + if goMod != null then commands else [ ]; + + sources = mapAttrs + (goPackagePath: meta: fetchGoModule { + goPackagePath = meta.replaced or goPackagePath; + inherit (meta) version hash; + inherit go; + }) + modulesStruct.mod; + in + runCommand "vendor-env" + { + nativeBuildInputs = [ go ]; + json = toJSON (filterAttrs (n: _: n != defaultPackage) modulesStruct.mod); + + sources = toJSON (filterAttrs (n: _: n != defaultPackage) sources); + + passthru = { + inherit sources; + }; + + passAsFile = [ "json" "sources" ]; + } + ( + '' + mkdir vendor + + export GOCACHE=$TMPDIR/go-cache + export GOPATH="$TMPDIR/go" + + ${internal.symlink} + ${concatStringsSep "\n" localReplaceCommands} + + mv vendor $out + '' + ); + + # Select Go attribute based on version specified in go.mod + selectGo = attrs: goMod: attrs.go or (if goMod == null then pkgs.go else + ( + let + goVersion = goMod.go; + goAttr = "go_" + (replaceStrings [ "." ] [ "_" ] goVersion); + in + ( + if hasAttr goAttr pkgs then pkgs.${goAttr} + else trace "go.mod specified Go version ${goVersion} but doesn't exist. Falling back to ${pkgs.go.version}." pkgs.go + ) + )); + + # Strip the rubbish that Go adds to versions, and fall back to a version based on the date if it's a placeholder value + stripVersion = version: + let + parts = elemAt (split "(\\+|-)" (removePrefix "v" version)); + v = parts 0; + d = parts 2; + in + if v != "0.0.0" then v else "unstable-" + (concatStringsSep "-" [ + (substring 0 4 d) + (substring 4 2 d) + (substring 6 2 d) + ]); + + mkGoEnv = + { pwd + }@attrs: + let + goMod = parseGoMod (readFile "${toString pwd}/go.mod"); + modulesStruct = fromTOML (readFile "${toString pwd}/gomod2nix.toml"); + + go = selectGo attrs goMod; + + vendorEnv = mkVendorEnv { + inherit go modulesStruct pwd goMod; + }; + + in + stdenv.mkDerivation (removeAttrs attrs [ "pwd" ] // { + name = "${baseNameOf goMod.module}-env"; + + dontUnpack = true; + dontConfigure = true; + dontInstall = true; + + propagatedNativeBuildInputs = [ go ]; + + GO_NO_VENDOR_CHECKS = "1"; + + GO111MODULE = "on"; + GOFLAGS = "-mod=vendor"; + + preferLocalBuild = true; + + buildPhase = '' + mkdir $out + + export GOCACHE=$TMPDIR/go-cache + export GOPATH="$out" + export GOSUMDB=off + export GOPROXY=off + + '' + optionalString (pathExists (pwd + "/tools.go")) '' + mkdir source + cp ${pwd + "/go.mod"} source/go.mod + cp ${pwd + "/go.sum"} source/go.sum + cp ${pwd + "/tools.go"} source/tools.go + cd source + ln -s ${vendorEnv} vendor + + ${internal.install} + ''; + }); + + buildGoApplication = + { modules ? pwd + "/gomod2nix.toml" + , src ? pwd + , pwd ? null + , nativeBuildInputs ? [ ] + , allowGoReference ? false + , meta ? { } + , passthru ? { } + , tags ? [ ] + + # needed for buildFlags{,Array} warning + , buildFlags ? "" + , buildFlagsArray ? "" + + , ... + }@attrs: + let + modulesStruct = fromTOML (readFile modules); + + goModPath = "${toString pwd}/go.mod"; + + goMod = + if pwd != null && pathExists goModPath + then parseGoMod (readFile goModPath) + else null; + + go = selectGo attrs goMod; + + removeReferences = [ ] ++ optional (!allowGoReference) go; + + defaultPackage = modulesStruct.goPackagePath or ""; + + vendorEnv = mkVendorEnv { + inherit go modulesStruct defaultPackage goMod pwd; + }; + + in + warnIf (buildFlags != "" || buildFlagsArray != "") + "Use the `ldflags` and/or `tags` attributes instead of `buildFlags`/`buildFlagsArray`" + stdenv.mkDerivation + (optionalAttrs (defaultPackage != "") + { + pname = attrs.pname or baseNameOf defaultPackage; + version = stripVersion (modulesStruct.mod.${defaultPackage}).version; + src = vendorEnv.passthru.sources.${defaultPackage}; + } // optionalAttrs (hasAttr "subPackages" modulesStruct) { + subPackages = modulesStruct.subPackages; + } // attrs // { + nativeBuildInputs = [ removeReferencesTo go ] ++ nativeBuildInputs; + + inherit (go) GOOS GOARCH; + + GO_NO_VENDOR_CHECKS = "1"; + + GO111MODULE = "on"; + GOFLAGS = "-mod=vendor"; + + configurePhase = attrs.configurePhase or '' + runHook preConfigure + + export GOCACHE=$TMPDIR/go-cache + export GOPATH="$TMPDIR/go" + export GOSUMDB=off + export GOPROXY=off + cd "$modRoot" + if [ -n "${vendorEnv}" ]; then + rm -rf vendor + ln -s ${vendorEnv} vendor + fi + + runHook postConfigure + ''; + + buildPhase = attrs.buildPhase or '' + runHook preBuild + + exclude='\(/_\|examples\|Godeps\|testdata' + if [[ -n "$excludedPackages" ]]; then + IFS=' ' read -r -a excludedArr <<<$excludedPackages + printf -v excludedAlternates '%s\\|' "''${excludedArr[@]}" + excludedAlternates=''${excludedAlternates%\\|} # drop final \| added by printf + exclude+='\|'"$excludedAlternates" + fi + exclude+='\)' + + buildGoDir() { + local d; local cmd; + cmd="$1" + d="$2" + . $TMPDIR/buildFlagsArray + local OUT + if ! OUT="$(go $cmd $buildFlags "''${buildFlagsArray[@]}" ''${tags:+-tags=${concatStringsSep "," tags}} ''${ldflags:+-ldflags="$ldflags"} -v -p $NIX_BUILD_CORES $d 2>&1)"; then + if echo "$OUT" | grep -qE 'imports .*?: no Go files in'; then + echo "$OUT" >&2 + return 1 + fi + if ! echo "$OUT" | grep -qE '(no( buildable| non-test)?|build constraints exclude all) Go (source )?files'; then + echo "$OUT" >&2 + return 1 + fi + fi + if [ -n "$OUT" ]; then + echo "$OUT" >&2 + fi + return 0 + } + + getGoDirs() { + local type; + type="$1" + if [ -n "$subPackages" ]; then + echo "$subPackages" | sed "s,\(^\| \),\1./,g" + else + find . -type f -name \*$type.go -exec dirname {} \; | grep -v "/vendor/" | sort --unique | grep -v "$exclude" + fi + } + + if (( "''${NIX_DEBUG:-0}" >= 1 )); then + buildFlagsArray+=(-x) + fi + + if [ ''${#buildFlagsArray[@]} -ne 0 ]; then + declare -p buildFlagsArray > $TMPDIR/buildFlagsArray + else + touch $TMPDIR/buildFlagsArray + fi + if [ -z "$enableParallelBuilding" ]; then + export NIX_BUILD_CORES=1 + fi + for pkg in $(getGoDirs ""); do + echo "Building subPackage $pkg" + buildGoDir install "$pkg" + done + '' + optionalString (stdenv.hostPlatform != stdenv.buildPlatform) '' + # normalize cross-compiled builds w.r.t. native builds + ( + dir=$GOPATH/bin/${go.GOOS}_${go.GOARCH} + if [[ -n "$(shopt -s nullglob; echo $dir/*)" ]]; then + mv $dir/* $dir/.. + fi + if [[ -d $dir ]]; then + rmdir $dir + fi + ) + '' + '' + runHook postBuild + ''; + + doCheck = attrs.doCheck or true; + checkPhase = attrs.checkPhase or '' + runHook preCheck + + for pkg in $(getGoDirs test); do + buildGoDir test $checkFlags "$pkg" + done + + runHook postCheck + ''; + + installPhase = attrs.installPhase or '' + runHook preInstall + + mkdir -p $out + dir="$GOPATH/bin" + [ -e "$dir" ] && cp -r $dir $out + + runHook postInstall + ''; + + preFixup = (attrs.preFixup or "") + '' + find $out/{bin,libexec,lib} -type f 2>/dev/null | xargs -r ${removeExpr removeReferences} || true + ''; + + strictDeps = true; + + disallowedReferences = optional (!allowGoReference) go; + + passthru = { inherit go vendorEnv; } // passthru; + + meta = { platforms = go.meta.platforms or platforms.all; } // meta; + }); + +in +{ + inherit buildGoApplication mkGoEnv; +} diff --git a/nix/builder/fetch.sh b/nix/builder/fetch.sh new file mode 100644 index 0000000..8c6bdb3 --- /dev/null +++ b/nix/builder/fetch.sh @@ -0,0 +1,13 @@ +source $stdenv/setup + +export HOME=$(mktemp -d) + +# Call once first outside of subshell for better error reporting +go mod download "$goPackagePath@$version" + +dir=$(go mod download --json "$goPackagePath@$version" | jq -r .Dir) + +chmod -R +w $dir +find $dir -iname ".ds_store" | xargs -r rm -rf + +cp -r $dir $out diff --git a/nix/builder/install/install.go b/nix/builder/install/install.go new file mode 100644 index 0000000..4f770b0 --- /dev/null +++ b/nix/builder/install/install.go @@ -0,0 +1,57 @@ +package main + +import ( + "fmt" + "go/parser" + "go/token" + "io" + "os" + "os/exec" + "strconv" +) + +const filename = "tools.go" + +func main() { + fset := token.NewFileSet() + + var src []byte + { + f, err := os.Open(filename) + if err != nil { + panic(err) + } + + src, err = io.ReadAll(f) + if err != nil { + panic(err) + } + } + + f, err := parser.ParseFile(fset, filename, src, parser.ImportsOnly) + if err != nil { + fmt.Println(err) + return + } + + for _, s := range f.Imports { + path, err := strconv.Unquote(s.Path.Value) + if err != nil { + panic(err) + } + + cmd := exec.Command("go", "install", path) + + fmt.Printf("Executing '%s'\n", cmd) + + err = cmd.Start() + if err != nil { + panic(err) + } + + err = cmd.Wait() + if err != nil { + panic(err) + } + } +} diff --git a/nix/builder/parser.nix b/nix/builder/parser.nix new file mode 100644 index 0000000..eb6f75e --- /dev/null +++ b/nix/builder/parser.nix @@ -0,0 +1,141 @@ +# Parse go.mod in Nix +# Returns a Nix structure with the contents of the go.mod passed in +# in normalised form. + +let + inherit (builtins) elemAt mapAttrs split foldl' match filter typeOf hasAttr length; + + # Strip lines with comments & other junk + stripStr = s: elemAt (split "^ *" (elemAt (split " *$" s) 0)) 2; + stripLines = initialLines: foldl' (acc: f: f acc) initialLines [ + # Strip comments + (lines: map + (l: stripStr (elemAt (splitString "//" l) 0)) + lines) + + # Strip leading tabs characters + (lines: map (l: elemAt (match "(\t)?(.*)" l) 1) lines) + + # Filter empty lines + (filter (l: l != "")) + ]; + + # Parse lines into a structure + parseLines = lines: (foldl' + (acc: l: + let + m = match "([^ )]*) *(.*)" l; + directive = elemAt m 0; + rest = elemAt m 1; + + # Maintain parser state (inside parens or not) + inDirective = + if rest == "(" then directive + else if rest == ")" then null + else acc.inDirective + ; + + in + { + data = (acc.data // ( + if directive == "" && rest == ")" then { } + else if inDirective != null && rest == "(" && ! hasAttr inDirective acc.data then { + ${inDirective} = { }; + } + else if rest == "(" || rest == ")" then { } + else if inDirective != null then { + ${inDirective} = acc.data.${inDirective} // { ${directive} = rest; }; + } else if directive == "replace" then + ( + let + segments = split " => " rest; + getSegment = elemAt segments; + in + assert length segments == 3; { + replace = acc.data.replace // { + ${getSegment 0} = "=> ${getSegment 2}"; + }; + } + ) + else { + ${directive} = rest; + } + ) + ); + inherit inDirective; + }) + { + inDirective = null; + data = { + require = { }; + replace = { }; + exclude = { }; + }; + } + lines + ).data; + + normaliseDirectives = data: ( + let + normaliseString = s: + let + m = builtins.match "([^ ]+) (.+)" s; + in + { + ${elemAt m 0} = elemAt m 1; + }; + require = data.require or { }; + replace = data.replace or { }; + exclude = data.exclude or { }; + in + data // { + require = + if typeOf require == "string" then normaliseString require + else require; + replace = + if typeOf replace == "string" then normaliseString replace + else replace; + } + ); + + parseVersion = ver: + let + m = elemAt (match "([^-]+)-?([^-]*)-?([^-]*)" ver); + v = elemAt (match "([^+]+)\\+?(.*)" (m 0)); + in + { + version = v 0; + versionSuffix = v 1; + date = m 1; + rev = m 2; + }; + + parseReplace = data: ( + data // { + replace = + mapAttrs + (_: v: + let + m = match "=> ([^ ]+) (.+)" v; + m2 = match "=> (.*+)" v; + in + if m != null then { + goPackagePath = elemAt m 0; + version = elemAt m 1; + } else { + path = elemAt m2 0; + }) + data.replace; + } + ); + + splitString = sep: s: filter (t: t != [ ]) (split sep s); + +in +contents: +foldl' (acc: f: f acc) (splitString "\n" contents) [ + stripLines + parseLines + normaliseDirectives + parseReplace +] diff --git a/nix/builder/symlink/symlink.go b/nix/builder/symlink/symlink.go new file mode 100644 index 0000000..3dbb383 --- /dev/null +++ b/nix/builder/symlink/symlink.go @@ -0,0 +1,110 @@ +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "sort" +) + +type Package struct { + GoPackagePath string `json:"-"` + Version string `json:"version"` + Hash string `json:"hash"` + ReplacedPath string `json:"replaced,omitempty"` +} + +// type Output struct { +// SchemaVersion int `json:"schema"` +// Mod map[string]*Package `json:"mod"` +// } + +func main() { + + // var output Output + sources := make(map[string]string) + pkgs := make(map[string]*Package) + + { + b, err := ioutil.ReadFile(os.Getenv("sourcesPath")) + if err != nil { + panic(err) + } + + err = json.Unmarshal(b, &sources) + if err != nil { + panic(err) + } + } + + { + b, err := ioutil.ReadFile(os.Getenv("jsonPath")) + if err != nil { + panic(err) + } + + err = json.Unmarshal(b, &pkgs) + if err != nil { + panic(err) + } + } + + keys := make([]string, 0, len(pkgs)) + for key := range pkgs { + keys = append(keys, key) + } + sort.Strings(keys) + + // Iterate, in reverse order + for i := len(keys) - 1; i >= 0; i-- { + key := keys[i] + src := sources[key] + + paths := []string{key} + + for _, path := range paths { + + vendorDir := filepath.Join("vendor", filepath.Dir(path)) + if err := os.MkdirAll(vendorDir, 0755); err != nil { + panic(err) + } + + if _, err := os.Stat(filepath.Join("vendor", path)); err == nil { + files, err := ioutil.ReadDir(src) + if err != nil { + panic(err) + } + + for _, f := range files { + innerSrc := filepath.Join(src, f.Name()) + dst := filepath.Join("vendor", path, f.Name()) + if err := os.Symlink(innerSrc, dst); err != nil { + // assume it's an existing directory, try to link the directory content instead. + // TODO should we do this recursively + files, err := ioutil.ReadDir(innerSrc) + if err != nil { + panic(err) + } + for _, f := range files { + if err := os.Symlink(filepath.Join(innerSrc, f.Name()), filepath.Join(dst, f.Name())); err != nil { + fmt.Println("ignore symlink error", filepath.Join(innerSrc, f.Name()), filepath.Join(dst, f.Name())) + } + } + } + } + + continue + } + + // If the file doesn't already exist, just create a simple symlink + err := os.Symlink(src, filepath.Join("vendor", path)) + if err != nil { + panic(err) + } + + } + } + +} -- 2.43.4 From b41630e941b44a8fffe09f42d9a7efb728fcbfa1 Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Tue, 19 Jul 2022 15:08:51 +0200 Subject: [PATCH 2/6] Embed static/ and templates/ --- default.nix | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/default.nix b/default.nix index 6c88213..336e58d 100644 --- a/default.nix +++ b/default.nix @@ -11,9 +11,8 @@ let }) ]; }; -in - pkgs.gomod.buildGoApplication { - pname = "bottin"; + bin = pkgs.gomod.buildGoApplication { + pname = "bottin-bin"; version = "0.1.0"; src = ./.; modules = ./gomod2nix.toml; @@ -26,5 +25,24 @@ in license = licenses.gpl3Plus; platforms = platforms.linux; }; - } + }; +in + pkgs.stdenv.mkDerivation { + pname = "bottin"; + version = "0.1.0"; + src = ./.; + installPhase = '' + mkdir -p $out/ + + cat > guichet < Date: Tue, 19 Jul 2022 15:39:49 +0200 Subject: [PATCH 3/6] Ease packaging by making resource path configurable --- admin.go | 8 ++++---- default.nix | 8 ++++---- directory.go | 4 ++-- invite.go | 8 ++++---- main.go | 36 +++++++++++++++++++++++++++++++++--- profile.go | 4 ++-- 6 files changed, 49 insertions(+), 19 deletions(-) diff --git a/admin.go b/admin.go index a2c70c5..3d6674e 100644 --- a/admin.go +++ b/admin.go @@ -48,7 +48,7 @@ type AdminUsersTplData struct { } func handleAdminUsers(w http.ResponseWriter, r *http.Request) { - templateAdminUsers := template.Must(template.ParseFiles("templates/layout.html", "templates/admin_users.html")) + templateAdminUsers := template.Must(template.ParseFiles(config.Resources[0]+"/templates/layout.html", config.Resources[0]+"/templates/admin_users.html")) login := checkAdminLogin(w, r) if login == nil { @@ -87,7 +87,7 @@ type AdminGroupsTplData struct { } func handleAdminGroups(w http.ResponseWriter, r *http.Request) { - templateAdminGroups := template.Must(template.ParseFiles("templates/layout.html", "templates/admin_groups.html")) + templateAdminGroups := template.Must(template.ParseFiles(config.Resources[0]+"/templates/layout.html", config.Resources[0]+"/templates/admin_groups.html")) login := checkAdminLogin(w, r) if login == nil { @@ -165,7 +165,7 @@ type PropValues struct { } func handleAdminLDAP(w http.ResponseWriter, r *http.Request) { - templateAdminLDAP := template.Must(template.ParseFiles("templates/layout.html", "templates/admin_ldap.html")) + templateAdminLDAP := template.Must(template.ParseFiles(config.Resources[0]+"/templates/layout.html", config.Resources[0]+"/templates/admin_ldap.html")) login := checkAdminLogin(w, r) if login == nil { @@ -551,7 +551,7 @@ type CreateData struct { } func handleAdminCreate(w http.ResponseWriter, r *http.Request) { - templateAdminCreate := template.Must(template.ParseFiles("templates/layout.html", "templates/admin_create.html")) + templateAdminCreate := template.Must(template.ParseFiles(config.Resources[0]+"/templates/layout.html", config.Resources[0]+"/templates/admin_create.html")) login := checkAdminLogin(w, r) if login == nil { diff --git a/default.nix b/default.nix index 336e58d..24b759c 100644 --- a/default.nix +++ b/default.nix @@ -12,7 +12,7 @@ let ]; }; bin = pkgs.gomod.buildGoApplication { - pname = "bottin-bin"; + pname = "guichet-bin"; version = "0.1.0"; src = ./.; modules = ./gomod2nix.toml; @@ -20,15 +20,15 @@ let CGO_ENABLED=0; meta = with pkgs.lib; { - description = "Bottin is a cloud-native LDAP server backed by a Consul datastore"; - homepage = "https://git.deuxfleurs.fr/Deuxfleurs/bottin"; + description = "Interface web pour gérer le LDAP: changer son mot de passe, ses infos de profil, inviter des gens, administration"; + homepage = "https://git.deuxfleurs.fr/Deuxfleurs/guichet"; license = licenses.gpl3Plus; platforms = platforms.linux; }; }; in pkgs.stdenv.mkDerivation { - pname = "bottin"; + pname = "guichet"; version = "0.1.0"; src = ./.; diff --git a/directory.go b/directory.go index ab5dea3..4f4a5a2 100644 --- a/directory.go +++ b/directory.go @@ -13,7 +13,7 @@ const FIELD_NAME_PROFILE_PICTURE = "profilePicture" const FIELD_NAME_DIRECTORY_VISIBILITY = "directoryVisibility" func handleDirectory(w http.ResponseWriter, r *http.Request) { - templateDirectory := template.Must(template.ParseFiles("templates/layout.html", "templates/directory.html")) + templateDirectory := template.Must(template.ParseFiles(config.Resources[0]+"/templates/layout.html", config.Resources[0]+"/templates/directory.html")) login := checkLogin(w, r) if login == nil { @@ -37,7 +37,7 @@ type SearchResults struct { } func handleDirectorySearch(w http.ResponseWriter, r *http.Request) { - templateDirectoryResults := template.Must(template.ParseFiles("templates/directory_results.html")) + templateDirectoryResults := template.Must(template.ParseFiles(config.Resources[0]+"/templates/directory_results.html")) //Get input value by user r.ParseMultipartForm(1024) diff --git a/invite.go b/invite.go index 689c7e4..2d01e44 100644 --- a/invite.go +++ b/invite.go @@ -60,7 +60,7 @@ func handleInvitationCode(w http.ResponseWriter, r *http.Request) { inviteDn := config.InvitationNameAttr + "=" + code_id + "," + config.InvitationBaseDN err := l.Bind(inviteDn, code_pw) if err != nil { - templateInviteInvalidCode := template.Must(template.ParseFiles("templates/layout.html", "templates/invite_invalid_code.html")) + templateInviteInvalidCode := template.Must(template.ParseFiles(config.Resources[0]+"/templates/layout.html", config.Resources[0]+"/templates/invite_invalid_code.html")) templateInviteInvalidCode.Execute(w, nil) return } @@ -110,7 +110,7 @@ type NewAccountData struct { } func handleNewAccount(w http.ResponseWriter, r *http.Request, l *ldap.Conn, invitedBy string) bool { - templateInviteNewAccount := template.Must(template.ParseFiles("templates/layout.html", "templates/invite_new_account.html")) + templateInviteNewAccount := template.Must(template.ParseFiles(config.Resources[0]+"/templates/layout.html", config.Resources[0]+"/templates/invite_new_account.html")) data := &NewAccountData{} @@ -239,7 +239,7 @@ type CodeMailFields struct { } func handleInviteSendCode(w http.ResponseWriter, r *http.Request) { - templateInviteSendCode := template.Must(template.ParseFiles("templates/layout.html", "templates/invite_send_code.html")) + templateInviteSendCode := template.Must(template.ParseFiles(config.Resources[0]+"/templates/layout.html", config.Resources[0]+"/templates/invite_send_code.html")) login := checkInviterLogin(w, r) if login == nil { @@ -298,7 +298,7 @@ func trySendCode(login *LoginStatus, choice string, sendto string, data *SendCod return } - templateMail := template.Must(template.ParseFiles("templates/invite_mail.txt")) + templateMail := template.Must(template.ParseFiles(config.Resources[0]+"/templates/invite_mail.txt")) buf := bytes.NewBuffer([]byte{}) templateMail.Execute(buf, &CodeMailFields{ To: sendto, diff --git a/main.go b/main.go index d574f3f..51a23aa 100644 --- a/main.go +++ b/main.go @@ -11,6 +11,7 @@ import ( "log" "net/http" "os" + "path/filepath" "strings" "github.com/go-ldap/ldap/v3" @@ -19,6 +20,7 @@ import ( ) type ConfigFile struct { + Resources []string `json:"resources"` HttpBindAddr string `json:"http_bind_addr"` LdapServerAddr string `json:"ldap_server_addr"` LdapTLS bool `json:"ldap_tls"` @@ -62,6 +64,7 @@ var store sessions.Store = nil func readConfig() ConfigFile { // Default configuration values for certain fields config_file := ConfigFile{ + Resources: []string{}, HttpBindAddr: ":9991", LdapServerAddr: "ldap://127.0.0.1:389", @@ -91,13 +94,40 @@ func readConfig() ConfigFile { log.Fatal(err) } + // Enrich the Resource entry with default values + config_file.Resources = append(config_file.Resources, ".") + ex, err := os.Executable() + if err == nil { + exPath := filepath.Dir(ex) + config_file.Resources = append(config_file.Resources, exPath) + } + fmt.Println(config_file.Resources) + return config_file } +func selectResource(conf *ConfigFile) { + ResourceLoop: + for _, p := range conf.Resources { + for _, suffix := range []string{"", "/templates", "/static"} { + _, err := os.Stat(p + suffix) + if err != nil { + continue ResourceLoop + } + } + + // Success, this Resource folder is the one! + conf.Resources = []string{p} + return + } + log.Fatalf("Unable to find templates/ and static/ in the following paths: %s", conf.Resources) +} + func main() { flag.Parse() config_file := readConfig() + selectResource(&config_file) config = &config_file session_key := make([]byte, 32) @@ -127,7 +157,7 @@ func main() { r.HandleFunc("/admin/ldap/{dn}", handleAdminLDAP) r.HandleFunc("/admin/create/{template}/{super_dn}", handleAdminCreate) - staticfiles := http.FileServer(http.Dir("static")) + staticfiles := http.FileServer(http.Dir(config.Resources[0]+"/static")) r.Handle("/static/{file:.*}", http.StripPrefix("/static/", staticfiles)) log.Printf("Starting HTTP server on %s", config.HttpBindAddr) @@ -296,7 +326,7 @@ type HomePageData struct { } func handleHome(w http.ResponseWriter, r *http.Request) { - templateHome := template.Must(template.ParseFiles("templates/layout.html", "templates/home.html")) + templateHome := template.Must(template.ParseFiles(config.Resources[0]+"/templates/layout.html", config.Resources[0]+"/templates/home.html")) login := checkLogin(w, r) if login == nil { @@ -338,7 +368,7 @@ type LoginFormData struct { } func handleLogin(w http.ResponseWriter, r *http.Request) *LoginInfo { - templateLogin := template.Must(template.ParseFiles("templates/layout.html", "templates/login.html")) + templateLogin := template.Must(template.ParseFiles(config.Resources[0]+"/templates/layout.html", config.Resources[0]+"/templates/login.html")) if r.Method == "GET" { templateLogin.Execute(w, LoginFormData{}) diff --git a/profile.go b/profile.go index 603b10a..51f0b08 100644 --- a/profile.go +++ b/profile.go @@ -22,7 +22,7 @@ type ProfileTplData struct { } func handleProfile(w http.ResponseWriter, r *http.Request) { - templateProfile := template.Must(template.ParseFiles("templates/layout.html", "templates/profile.html")) + templateProfile := template.Must(template.ParseFiles(config.Resources[0]+"/templates/layout.html", config.Resources[0]+"/templates/profile.html")) login := checkLogin(w, r) if login == nil { @@ -97,7 +97,7 @@ type PasswdTplData struct { } func handlePasswd(w http.ResponseWriter, r *http.Request) { - templatePasswd := template.Must(template.ParseFiles("templates/layout.html", "templates/passwd.html")) + templatePasswd := template.Must(template.ParseFiles(config.Resources[0]+"/templates/layout.html", config.Resources[0]+"/templates/passwd.html")) login := checkLogin(w, r) if login == nil { -- 2.43.4 From 158eda06de31a52f15712fa0e5695054d15dfe3a Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Tue, 19 Jul 2022 15:53:39 +0200 Subject: [PATCH 4/6] Format + documentation --- README.md | 9 +++++++++ default.nix | 22 +++++++++------------- directory.go | 2 +- invite.go | 4 ++-- main.go | 19 +++++++++---------- profile.go | 2 +- 6 files changed, 31 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index fad0594..9751fab 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,15 @@ Guichet requires go 1.13 or later. To build Guichet, clone this repository outside of your `$GOPATH`. Then, run `make` in the root of the repo. +## Releasing Guichet + +To build Guichet in a controlled environment, because you plan to release it for example, please use Nix. + +```bash +nix-build -A bin # build only the Go binary +nix-build -A pkg # build the binary and add the ressources +nix-build -A docker # build a docker container +``` ## Configuration of Guichet diff --git a/default.nix b/default.nix index 24b759c..cd17475 100644 --- a/default.nix +++ b/default.nix @@ -11,7 +11,8 @@ let }) ]; }; - bin = pkgs.gomod.buildGoApplication { +in rec { + bin = pkgs.gomod.buildGoApplication { pname = "guichet-bin"; version = "0.1.0"; src = ./.; @@ -26,23 +27,18 @@ let platforms = platforms.linux; }; }; -in - pkgs.stdenv.mkDerivation { + pkg = pkgs.stdenv.mkDerivation { pname = "guichet"; version = "0.1.0"; src = ./.; installPhase = '' mkdir -p $out/ - - cat > guichet < Date: Tue, 19 Jul 2022 16:00:22 +0200 Subject: [PATCH 5/6] Add docker entry --- README.md | 7 +++++++ default.nix | 9 ++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9751fab..449017f 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,13 @@ nix-build -A pkg # build the binary and add the ressources nix-build -A docker # build a docker container ``` +Publishing the container: + +```bash +docker load < $(nix-build -A docker) +docker push ??? +``` + ## Configuration of Guichet Guichet is configured using a simple JSON config file which is a dictionnary whose keys diff --git a/default.nix b/default.nix index cd17475..3cafd8c 100644 --- a/default.nix +++ b/default.nix @@ -38,7 +38,10 @@ in rec { cp -r templates static $out/ ''; }; - /*docker = pkgs.xxx { - - };*/ + docker = pkgs.dockerTools.buildImage { + name = "dxflrs/guichet"; + config = { + Cmd = [ "${pkg}/guichet" ]; + }; + }; } -- 2.43.4 From 11574aeb19cd971b20d37353111c4f37ffdfb7f5 Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Sun, 24 Jul 2022 10:07:41 +0200 Subject: [PATCH 6/6] Remove vendoring of gomod2nix --- default.nix | 8 +- nix/builder/default.nix | 387 --------------------------------- nix/builder/fetch.sh | 13 -- nix/builder/install/install.go | 57 ----- nix/builder/parser.nix | 141 ------------ nix/builder/symlink/symlink.go | 110 ---------- 6 files changed, 7 insertions(+), 709 deletions(-) delete mode 100644 nix/builder/default.nix delete mode 100644 nix/builder/fetch.sh delete mode 100644 nix/builder/install/install.go delete mode 100644 nix/builder/parser.nix delete mode 100644 nix/builder/symlink/symlink.go diff --git a/default.nix b/default.nix index 3cafd8c..cf40d93 100644 --- a/default.nix +++ b/default.nix @@ -4,10 +4,16 @@ let url = "https://github.com/NixOS/nixpkgs/archive/d2db10786f27619d5519b12b03fb10dc8ca95e59.tar.gz"; sha256 = "0s9gigs3ylnq5b94rfcmxvrmmr3kzhs497gksajf638d5bv7zcl5"; }; + gomod2nix = fetchGit { + url = "https://github.com/tweag/gomod2nix.git"; + ref = "master"; + rev = "40d32f82fc60d66402eb0972e6e368aeab3faf58"; + }; + pkgs = import pkgsSrc { overlays = [ (self: super: { - gomod = super.callPackage ./nix/builder { }; + gomod = super.callPackage "${gomod2nix}/builder/" { }; }) ]; }; diff --git a/nix/builder/default.nix b/nix/builder/default.nix deleted file mode 100644 index 8b8ae19..0000000 --- a/nix/builder/default.nix +++ /dev/null @@ -1,387 +0,0 @@ -{ stdenv -, stdenvNoCC -, runCommand -, buildEnv -, lib -, fetchgit -, removeReferencesTo -, jq -, cacert -, pkgs -, pkgsBuildBuild -}: -let - - inherit (builtins) substring toJSON hasAttr trace split readFile elemAt; - inherit (lib) - concatStringsSep replaceStrings removePrefix optionalString pathExists - optional concatMapStrings fetchers filterAttrs mapAttrs mapAttrsToList - warnIf optionalAttrs platforms - ; - - parseGoMod = import ./parser.nix; - - removeExpr = refs: ''remove-references-to ${concatMapStrings (ref: " -t ${ref}") refs}''; - - # Internal only build-time attributes - internal = - let - mkInternalPkg = name: src: pkgsBuildBuild.runCommand "gomod2nix-${name}" - { - inherit (pkgsBuildBuild.go) GOOS GOARCH; - nativeBuildInputs = [ pkgsBuildBuild.go ]; - } '' - export HOME=$(mktemp -d) - cp ${src} src.go - go build -o $out src.go - ''; - in - { - - # Create a symlink tree of vendored sources - symlink = mkInternalPkg "symlink" ./symlink/symlink.go; - - # Install development dependencies from tools.go - install = mkInternalPkg "symlink" ./install/install.go; - - }; - - fetchGoModule = - { hash - , goPackagePath - , version - , go ? pkgs.go - }: - stdenvNoCC.mkDerivation { - name = "${baseNameOf goPackagePath}_${version}"; - builder = ./fetch.sh; - inherit goPackagePath version; - nativeBuildInputs = [ go jq ]; - outputHashMode = "recursive"; - outputHashAlgo = null; - outputHash = hash; - SSL_CERT_FILE = "${cacert}/etc/ssl/certs/ca-bundle.crt"; - impureEnvVars = fetchers.proxyImpureEnvVars ++ [ "GOPROXY" ]; - }; - - mkVendorEnv = - { go - , modulesStruct - , localReplaceCommands ? [ ] - , defaultPackage ? "" - , goMod - , pwd - }: - let - localReplaceCommands = - let - localReplaceAttrs = filterAttrs (n: v: hasAttr "path" v) goMod.replace; - commands = ( - mapAttrsToList - (name: value: ( - '' - mkdir -p $(dirname vendor/${name}) - ln -s ${pwd + "/${value.path}"} vendor/${name} - '' - )) - localReplaceAttrs); - in - if goMod != null then commands else [ ]; - - sources = mapAttrs - (goPackagePath: meta: fetchGoModule { - goPackagePath = meta.replaced or goPackagePath; - inherit (meta) version hash; - inherit go; - }) - modulesStruct.mod; - in - runCommand "vendor-env" - { - nativeBuildInputs = [ go ]; - json = toJSON (filterAttrs (n: _: n != defaultPackage) modulesStruct.mod); - - sources = toJSON (filterAttrs (n: _: n != defaultPackage) sources); - - passthru = { - inherit sources; - }; - - passAsFile = [ "json" "sources" ]; - } - ( - '' - mkdir vendor - - export GOCACHE=$TMPDIR/go-cache - export GOPATH="$TMPDIR/go" - - ${internal.symlink} - ${concatStringsSep "\n" localReplaceCommands} - - mv vendor $out - '' - ); - - # Select Go attribute based on version specified in go.mod - selectGo = attrs: goMod: attrs.go or (if goMod == null then pkgs.go else - ( - let - goVersion = goMod.go; - goAttr = "go_" + (replaceStrings [ "." ] [ "_" ] goVersion); - in - ( - if hasAttr goAttr pkgs then pkgs.${goAttr} - else trace "go.mod specified Go version ${goVersion} but doesn't exist. Falling back to ${pkgs.go.version}." pkgs.go - ) - )); - - # Strip the rubbish that Go adds to versions, and fall back to a version based on the date if it's a placeholder value - stripVersion = version: - let - parts = elemAt (split "(\\+|-)" (removePrefix "v" version)); - v = parts 0; - d = parts 2; - in - if v != "0.0.0" then v else "unstable-" + (concatStringsSep "-" [ - (substring 0 4 d) - (substring 4 2 d) - (substring 6 2 d) - ]); - - mkGoEnv = - { pwd - }@attrs: - let - goMod = parseGoMod (readFile "${toString pwd}/go.mod"); - modulesStruct = fromTOML (readFile "${toString pwd}/gomod2nix.toml"); - - go = selectGo attrs goMod; - - vendorEnv = mkVendorEnv { - inherit go modulesStruct pwd goMod; - }; - - in - stdenv.mkDerivation (removeAttrs attrs [ "pwd" ] // { - name = "${baseNameOf goMod.module}-env"; - - dontUnpack = true; - dontConfigure = true; - dontInstall = true; - - propagatedNativeBuildInputs = [ go ]; - - GO_NO_VENDOR_CHECKS = "1"; - - GO111MODULE = "on"; - GOFLAGS = "-mod=vendor"; - - preferLocalBuild = true; - - buildPhase = '' - mkdir $out - - export GOCACHE=$TMPDIR/go-cache - export GOPATH="$out" - export GOSUMDB=off - export GOPROXY=off - - '' + optionalString (pathExists (pwd + "/tools.go")) '' - mkdir source - cp ${pwd + "/go.mod"} source/go.mod - cp ${pwd + "/go.sum"} source/go.sum - cp ${pwd + "/tools.go"} source/tools.go - cd source - ln -s ${vendorEnv} vendor - - ${internal.install} - ''; - }); - - buildGoApplication = - { modules ? pwd + "/gomod2nix.toml" - , src ? pwd - , pwd ? null - , nativeBuildInputs ? [ ] - , allowGoReference ? false - , meta ? { } - , passthru ? { } - , tags ? [ ] - - # needed for buildFlags{,Array} warning - , buildFlags ? "" - , buildFlagsArray ? "" - - , ... - }@attrs: - let - modulesStruct = fromTOML (readFile modules); - - goModPath = "${toString pwd}/go.mod"; - - goMod = - if pwd != null && pathExists goModPath - then parseGoMod (readFile goModPath) - else null; - - go = selectGo attrs goMod; - - removeReferences = [ ] ++ optional (!allowGoReference) go; - - defaultPackage = modulesStruct.goPackagePath or ""; - - vendorEnv = mkVendorEnv { - inherit go modulesStruct defaultPackage goMod pwd; - }; - - in - warnIf (buildFlags != "" || buildFlagsArray != "") - "Use the `ldflags` and/or `tags` attributes instead of `buildFlags`/`buildFlagsArray`" - stdenv.mkDerivation - (optionalAttrs (defaultPackage != "") - { - pname = attrs.pname or baseNameOf defaultPackage; - version = stripVersion (modulesStruct.mod.${defaultPackage}).version; - src = vendorEnv.passthru.sources.${defaultPackage}; - } // optionalAttrs (hasAttr "subPackages" modulesStruct) { - subPackages = modulesStruct.subPackages; - } // attrs // { - nativeBuildInputs = [ removeReferencesTo go ] ++ nativeBuildInputs; - - inherit (go) GOOS GOARCH; - - GO_NO_VENDOR_CHECKS = "1"; - - GO111MODULE = "on"; - GOFLAGS = "-mod=vendor"; - - configurePhase = attrs.configurePhase or '' - runHook preConfigure - - export GOCACHE=$TMPDIR/go-cache - export GOPATH="$TMPDIR/go" - export GOSUMDB=off - export GOPROXY=off - cd "$modRoot" - if [ -n "${vendorEnv}" ]; then - rm -rf vendor - ln -s ${vendorEnv} vendor - fi - - runHook postConfigure - ''; - - buildPhase = attrs.buildPhase or '' - runHook preBuild - - exclude='\(/_\|examples\|Godeps\|testdata' - if [[ -n "$excludedPackages" ]]; then - IFS=' ' read -r -a excludedArr <<<$excludedPackages - printf -v excludedAlternates '%s\\|' "''${excludedArr[@]}" - excludedAlternates=''${excludedAlternates%\\|} # drop final \| added by printf - exclude+='\|'"$excludedAlternates" - fi - exclude+='\)' - - buildGoDir() { - local d; local cmd; - cmd="$1" - d="$2" - . $TMPDIR/buildFlagsArray - local OUT - if ! OUT="$(go $cmd $buildFlags "''${buildFlagsArray[@]}" ''${tags:+-tags=${concatStringsSep "," tags}} ''${ldflags:+-ldflags="$ldflags"} -v -p $NIX_BUILD_CORES $d 2>&1)"; then - if echo "$OUT" | grep -qE 'imports .*?: no Go files in'; then - echo "$OUT" >&2 - return 1 - fi - if ! echo "$OUT" | grep -qE '(no( buildable| non-test)?|build constraints exclude all) Go (source )?files'; then - echo "$OUT" >&2 - return 1 - fi - fi - if [ -n "$OUT" ]; then - echo "$OUT" >&2 - fi - return 0 - } - - getGoDirs() { - local type; - type="$1" - if [ -n "$subPackages" ]; then - echo "$subPackages" | sed "s,\(^\| \),\1./,g" - else - find . -type f -name \*$type.go -exec dirname {} \; | grep -v "/vendor/" | sort --unique | grep -v "$exclude" - fi - } - - if (( "''${NIX_DEBUG:-0}" >= 1 )); then - buildFlagsArray+=(-x) - fi - - if [ ''${#buildFlagsArray[@]} -ne 0 ]; then - declare -p buildFlagsArray > $TMPDIR/buildFlagsArray - else - touch $TMPDIR/buildFlagsArray - fi - if [ -z "$enableParallelBuilding" ]; then - export NIX_BUILD_CORES=1 - fi - for pkg in $(getGoDirs ""); do - echo "Building subPackage $pkg" - buildGoDir install "$pkg" - done - '' + optionalString (stdenv.hostPlatform != stdenv.buildPlatform) '' - # normalize cross-compiled builds w.r.t. native builds - ( - dir=$GOPATH/bin/${go.GOOS}_${go.GOARCH} - if [[ -n "$(shopt -s nullglob; echo $dir/*)" ]]; then - mv $dir/* $dir/.. - fi - if [[ -d $dir ]]; then - rmdir $dir - fi - ) - '' + '' - runHook postBuild - ''; - - doCheck = attrs.doCheck or true; - checkPhase = attrs.checkPhase or '' - runHook preCheck - - for pkg in $(getGoDirs test); do - buildGoDir test $checkFlags "$pkg" - done - - runHook postCheck - ''; - - installPhase = attrs.installPhase or '' - runHook preInstall - - mkdir -p $out - dir="$GOPATH/bin" - [ -e "$dir" ] && cp -r $dir $out - - runHook postInstall - ''; - - preFixup = (attrs.preFixup or "") + '' - find $out/{bin,libexec,lib} -type f 2>/dev/null | xargs -r ${removeExpr removeReferences} || true - ''; - - strictDeps = true; - - disallowedReferences = optional (!allowGoReference) go; - - passthru = { inherit go vendorEnv; } // passthru; - - meta = { platforms = go.meta.platforms or platforms.all; } // meta; - }); - -in -{ - inherit buildGoApplication mkGoEnv; -} diff --git a/nix/builder/fetch.sh b/nix/builder/fetch.sh deleted file mode 100644 index 8c6bdb3..0000000 --- a/nix/builder/fetch.sh +++ /dev/null @@ -1,13 +0,0 @@ -source $stdenv/setup - -export HOME=$(mktemp -d) - -# Call once first outside of subshell for better error reporting -go mod download "$goPackagePath@$version" - -dir=$(go mod download --json "$goPackagePath@$version" | jq -r .Dir) - -chmod -R +w $dir -find $dir -iname ".ds_store" | xargs -r rm -rf - -cp -r $dir $out diff --git a/nix/builder/install/install.go b/nix/builder/install/install.go deleted file mode 100644 index 4f770b0..0000000 --- a/nix/builder/install/install.go +++ /dev/null @@ -1,57 +0,0 @@ -package main - -import ( - "fmt" - "go/parser" - "go/token" - "io" - "os" - "os/exec" - "strconv" -) - -const filename = "tools.go" - -func main() { - fset := token.NewFileSet() - - var src []byte - { - f, err := os.Open(filename) - if err != nil { - panic(err) - } - - src, err = io.ReadAll(f) - if err != nil { - panic(err) - } - } - - f, err := parser.ParseFile(fset, filename, src, parser.ImportsOnly) - if err != nil { - fmt.Println(err) - return - } - - for _, s := range f.Imports { - path, err := strconv.Unquote(s.Path.Value) - if err != nil { - panic(err) - } - - cmd := exec.Command("go", "install", path) - - fmt.Printf("Executing '%s'\n", cmd) - - err = cmd.Start() - if err != nil { - panic(err) - } - - err = cmd.Wait() - if err != nil { - panic(err) - } - } -} diff --git a/nix/builder/parser.nix b/nix/builder/parser.nix deleted file mode 100644 index eb6f75e..0000000 --- a/nix/builder/parser.nix +++ /dev/null @@ -1,141 +0,0 @@ -# Parse go.mod in Nix -# Returns a Nix structure with the contents of the go.mod passed in -# in normalised form. - -let - inherit (builtins) elemAt mapAttrs split foldl' match filter typeOf hasAttr length; - - # Strip lines with comments & other junk - stripStr = s: elemAt (split "^ *" (elemAt (split " *$" s) 0)) 2; - stripLines = initialLines: foldl' (acc: f: f acc) initialLines [ - # Strip comments - (lines: map - (l: stripStr (elemAt (splitString "//" l) 0)) - lines) - - # Strip leading tabs characters - (lines: map (l: elemAt (match "(\t)?(.*)" l) 1) lines) - - # Filter empty lines - (filter (l: l != "")) - ]; - - # Parse lines into a structure - parseLines = lines: (foldl' - (acc: l: - let - m = match "([^ )]*) *(.*)" l; - directive = elemAt m 0; - rest = elemAt m 1; - - # Maintain parser state (inside parens or not) - inDirective = - if rest == "(" then directive - else if rest == ")" then null - else acc.inDirective - ; - - in - { - data = (acc.data // ( - if directive == "" && rest == ")" then { } - else if inDirective != null && rest == "(" && ! hasAttr inDirective acc.data then { - ${inDirective} = { }; - } - else if rest == "(" || rest == ")" then { } - else if inDirective != null then { - ${inDirective} = acc.data.${inDirective} // { ${directive} = rest; }; - } else if directive == "replace" then - ( - let - segments = split " => " rest; - getSegment = elemAt segments; - in - assert length segments == 3; { - replace = acc.data.replace // { - ${getSegment 0} = "=> ${getSegment 2}"; - }; - } - ) - else { - ${directive} = rest; - } - ) - ); - inherit inDirective; - }) - { - inDirective = null; - data = { - require = { }; - replace = { }; - exclude = { }; - }; - } - lines - ).data; - - normaliseDirectives = data: ( - let - normaliseString = s: - let - m = builtins.match "([^ ]+) (.+)" s; - in - { - ${elemAt m 0} = elemAt m 1; - }; - require = data.require or { }; - replace = data.replace or { }; - exclude = data.exclude or { }; - in - data // { - require = - if typeOf require == "string" then normaliseString require - else require; - replace = - if typeOf replace == "string" then normaliseString replace - else replace; - } - ); - - parseVersion = ver: - let - m = elemAt (match "([^-]+)-?([^-]*)-?([^-]*)" ver); - v = elemAt (match "([^+]+)\\+?(.*)" (m 0)); - in - { - version = v 0; - versionSuffix = v 1; - date = m 1; - rev = m 2; - }; - - parseReplace = data: ( - data // { - replace = - mapAttrs - (_: v: - let - m = match "=> ([^ ]+) (.+)" v; - m2 = match "=> (.*+)" v; - in - if m != null then { - goPackagePath = elemAt m 0; - version = elemAt m 1; - } else { - path = elemAt m2 0; - }) - data.replace; - } - ); - - splitString = sep: s: filter (t: t != [ ]) (split sep s); - -in -contents: -foldl' (acc: f: f acc) (splitString "\n" contents) [ - stripLines - parseLines - normaliseDirectives - parseReplace -] diff --git a/nix/builder/symlink/symlink.go b/nix/builder/symlink/symlink.go deleted file mode 100644 index 3dbb383..0000000 --- a/nix/builder/symlink/symlink.go +++ /dev/null @@ -1,110 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "sort" -) - -type Package struct { - GoPackagePath string `json:"-"` - Version string `json:"version"` - Hash string `json:"hash"` - ReplacedPath string `json:"replaced,omitempty"` -} - -// type Output struct { -// SchemaVersion int `json:"schema"` -// Mod map[string]*Package `json:"mod"` -// } - -func main() { - - // var output Output - sources := make(map[string]string) - pkgs := make(map[string]*Package) - - { - b, err := ioutil.ReadFile(os.Getenv("sourcesPath")) - if err != nil { - panic(err) - } - - err = json.Unmarshal(b, &sources) - if err != nil { - panic(err) - } - } - - { - b, err := ioutil.ReadFile(os.Getenv("jsonPath")) - if err != nil { - panic(err) - } - - err = json.Unmarshal(b, &pkgs) - if err != nil { - panic(err) - } - } - - keys := make([]string, 0, len(pkgs)) - for key := range pkgs { - keys = append(keys, key) - } - sort.Strings(keys) - - // Iterate, in reverse order - for i := len(keys) - 1; i >= 0; i-- { - key := keys[i] - src := sources[key] - - paths := []string{key} - - for _, path := range paths { - - vendorDir := filepath.Join("vendor", filepath.Dir(path)) - if err := os.MkdirAll(vendorDir, 0755); err != nil { - panic(err) - } - - if _, err := os.Stat(filepath.Join("vendor", path)); err == nil { - files, err := ioutil.ReadDir(src) - if err != nil { - panic(err) - } - - for _, f := range files { - innerSrc := filepath.Join(src, f.Name()) - dst := filepath.Join("vendor", path, f.Name()) - if err := os.Symlink(innerSrc, dst); err != nil { - // assume it's an existing directory, try to link the directory content instead. - // TODO should we do this recursively - files, err := ioutil.ReadDir(innerSrc) - if err != nil { - panic(err) - } - for _, f := range files { - if err := os.Symlink(filepath.Join(innerSrc, f.Name()), filepath.Join(dst, f.Name())); err != nil { - fmt.Println("ignore symlink error", filepath.Join(innerSrc, f.Name()), filepath.Join(dst, f.Name())) - } - } - } - } - - continue - } - - // If the file doesn't already exist, just create a simple symlink - err := os.Symlink(src, filepath.Join("vendor", path)) - if err != nil { - panic(err) - } - - } - } - -} -- 2.43.4