[nix-improvements] modernize Nix infrastructure

.drone.yml

@@ -9,7 +9,7 @@ steps:
   - name: check formatting
     image: nixpkgs/nix:nixos-22.05
     commands:
-      - nix-shell --attr rust --run "cargo fmt -- --check"
+      - nix-shell --attr devShell --run "cargo fmt -- --check"
 
   - name: build
     image: nixpkgs/nix:nixos-22.05
@@ -40,7 +40,7 @@ steps:
     image: nixpkgs/nix:nixos-22.05
     commands:
       - nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
-      - nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
+      - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
 
 trigger:
   event:
@@ -63,17 +63,17 @@ steps:
     image: nixpkgs/nix:nixos-22.05
     commands:
       - nix-build --no-build-output --attr pkgs.amd64.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
-      - nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage"
+      - nix-shell --attr ci --run "./script/not-dynamic.sh result-bin/bin/garage"
 
   - name: integration tests
     image: nixpkgs/nix:nixos-22.05
     commands:
-      - nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
+      - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
 
   - name: upgrade tests
     image: nixpkgs/nix:nixos-22.05
     commands:
-      - nix-shell --attr integration --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false)
+      - nix-shell --attr ci --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false)
 
   - name: push static binary
     image: nixpkgs/nix:nixos-22.05
@@ -84,7 +84,7 @@ steps:
         from_secret: garagehq_aws_secret_access_key
       TARGET: "x86_64-unknown-linux-musl"
     commands:
-      - nix-shell --attr release --run "to_s3"
+      - nix-shell --attr ci --run "to_s3"
 
   - name: docker build and publish
     image: nixpkgs/nix:nixos-22.05
@@ -98,7 +98,7 @@ steps:
       - mkdir -p /kaniko/.docker
       - echo $DOCKER_AUTH > /kaniko/.docker/config.json
       - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
-      - nix-shell --attr release --run "to_docker"
+      - nix-shell --attr ci --run "to_docker"
 
 
 trigger:
@@ -119,17 +119,17 @@ steps:
     image: nixpkgs/nix:nixos-22.05
     commands:
       - nix-build --no-build-output --attr pkgs.i386.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
-      - nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage"
+      - nix-shell --attr ci --run "./script/not-dynamic.sh result-bin/bin/garage"
 
   - name: integration tests
     image: nixpkgs/nix:nixos-22.05
     commands:
-      - nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
+      - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
 
   - name: upgrade tests
     image: nixpkgs/nix:nixos-22.05
     commands:
-      - nix-shell --attr integration --run "./script/test-upgrade.sh v0.8.4 i686-unknown-linux-musl" || (cat /tmp/garage.log; false)
+      - nix-shell --attr ci --run "./script/test-upgrade.sh v0.8.4 i686-unknown-linux-musl" || (cat /tmp/garage.log; false)
 
   - name: push static binary
     image: nixpkgs/nix:nixos-22.05
@@ -140,7 +140,7 @@ steps:
         from_secret: garagehq_aws_secret_access_key
       TARGET: "i686-unknown-linux-musl"
     commands:
-      - nix-shell --attr release --run "to_s3"
+      - nix-shell --attr ci --run "to_s3"
 
   - name: docker build and publish
     image: nixpkgs/nix:nixos-22.05
@@ -154,7 +154,7 @@ steps:
       - mkdir -p /kaniko/.docker
       - echo $DOCKER_AUTH > /kaniko/.docker/config.json
       - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
-      - nix-shell --attr release --run "to_docker"
+      - nix-shell --attr ci --run "to_docker"
 
 trigger:
   event:
@@ -174,7 +174,7 @@ steps:
     image: nixpkgs/nix:nixos-22.05
     commands:
       - nix-build --no-build-output --attr pkgs.arm64.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
-      - nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage"
+      - nix-shell --attr ci --run "./script/not-dynamic.sh result-bin/bin/garage"
 
   - name: push static binary
     image: nixpkgs/nix:nixos-22.05
@@ -185,7 +185,7 @@ steps:
         from_secret: garagehq_aws_secret_access_key
       TARGET: "aarch64-unknown-linux-musl"
     commands:
-      - nix-shell --attr release --run "to_s3"
+      - nix-shell --attr ci --run "to_s3"
 
   - name: docker build and publish
     image: nixpkgs/nix:nixos-22.05
@@ -199,7 +199,7 @@ steps:
       - mkdir -p /kaniko/.docker
       - echo $DOCKER_AUTH > /kaniko/.docker/config.json
       - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
-      - nix-shell --attr release --run "to_docker"
+      - nix-shell --attr ci --run "to_docker"
 
 trigger:
   event:
@@ -219,7 +219,7 @@ steps:
     image: nixpkgs/nix:nixos-22.05
     commands:
       - nix-build --no-build-output --attr pkgs.arm.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
-      - nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage"
+      - nix-shell --attr ci --run "./script/not-dynamic.sh result-bin/bin/garage"
 
   - name: push static binary
     image: nixpkgs/nix:nixos-22.05
@@ -230,7 +230,7 @@ steps:
         from_secret: garagehq_aws_secret_access_key
       TARGET: "armv6l-unknown-linux-musleabihf"
     commands:
-      - nix-shell --attr release --run "to_s3"
+      - nix-shell --attr ci --run "to_s3"
 
   - name: docker build and publish
     image: nixpkgs/nix:nixos-22.05
@@ -244,7 +244,7 @@ steps:
       - mkdir -p /kaniko/.docker
       - echo $DOCKER_AUTH > /kaniko/.docker/config.json
       - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
-      - nix-shell --attr release --run "to_docker"
+      - nix-shell --attr ci --run "to_docker"
 
 trigger:
   event:
@@ -270,7 +270,7 @@ steps:
       - mkdir -p /root/.docker
       - echo $DOCKER_AUTH > /root/.docker/config.json
       - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
-      - nix-shell --attr release --run "multiarch_docker"
+      - nix-shell --attr ci --run "multiarch_docker"
   - name: refresh-index
     image: nixpkgs/nix:nixos-22.05
     environment:
@@ -280,7 +280,7 @@ steps:
         from_secret: garagehq_aws_secret_access_key
     commands:
       - mkdir -p /etc/nix && cp nix/nix.conf /etc/nix/nix.conf
-      - nix-shell --attr release --run "refresh_index"
+      - nix-shell --attr ci --run "refresh_index"
 
 depends_on:
   - release-linux-amd64
@@ -295,6 +295,6 @@ trigger:
 
 ---
 kind: signature
-hmac: 0c4b57eb4b27b7c6a6ff21ab87f0767fe3eb90f5d95d5cbcdccf794e9d2a5d86
+hmac: 9ff9f2ad9387b12ee49a327991a5373f01e40d28e0aeef3a13d3b741d8eb3c53
 
 ...

flake.nix

@@ -33,27 +33,57 @@
       compile = import ./nix/compile.nix;
     in
     flake-utils.lib.eachDefaultSystem (system:
-      let pkgs = nixpkgs.legacyPackages.${system};
+      let
-      in {
+        pkgs = nixpkgs.legacyPackages.${system};
-        packages = {
+      in
-          default = (compile {
+      {
-            inherit system git_version;
+        packages =
-            pkgsSrc = nixpkgs;
+          let
-            cargo2nixOverlay = cargo2nix.overlays.default;
+            packageFor = target: (compile {
-            release = true;
+              inherit system git_version target;
-          }).workspace.garage { compileMode = "build"; };
+              pkgsSrc = nixpkgs;
-        };
+              cargo2nixOverlay = cargo2nix.overlays.default;
-        devShell = (compile {
+              release = true;
-          inherit system git_version;
+            }).workspace.garage { compileMode = "build"; };
-          pkgsSrc = nixpkgs;
+          in
-          cargo2nixOverlay = cargo2nix.overlays.default;
+          {
-          release = false;
+            # default = native release build
-        }).workspaceShell { packages = with pkgs; [
+            default = packageFor null;
-          cargo-audit
+            # other = cross-compiled, statically-linked builds
-          cargo-outdated
+            amd64 = packageFor "x86_64-unknown-linux-musl";
-          rustfmt
+            i386 = packageFor "i686-unknown-linux-musl";
-          clang
+            arm64 = packageFor "aarch64-unknown-linux-musl";
-          mold
+            arm = packageFor "armv6l-unknown-linux-musl";
-        ]; };
+          };
+
+        # ---- developpment shell, for making native builds only ----
+        devShells =
+          let
+            shellWithPackages = (packages: (compile {
+              inherit system git_version;
+              pkgsSrc = nixpkgs;
+              cargo2nixOverlay = cargo2nix.overlays.default;
+            }).workspaceShell { inherit packages; });
+          in
+          {
+            default = shellWithPackages
+              (with pkgs; [
+                rustfmt
+                clang
+                mold
+              ]);
+
+            # import the full shell using `nix develop .#full`
+            full = shellWithPackages (with pkgs; [
+              rustfmt
+              clang
+              mold
+              # ---- extra packages for dev tasks ----
+              cargo-audit
+              cargo-outdated
+              cargo-machete
+              nixpkgs-fmt
+            ]);
+          };
       });
 }

nix/common.nix

@@ -14,4 +14,5 @@ rec {
   pkgsSrc = flake.defaultNix.inputs.nixpkgs;
   cargo2nix = flake.defaultNix.inputs.cargo2nix;
   cargo2nixOverlay = cargo2nix.overlays.default;
+  devShells = builtins.getAttr builtins.currentSystem flake.defaultNix.devShells;
 }

shell.nix

@@ -5,87 +5,36 @@ with import ./nix/common.nix;
 let
   pkgs = import pkgsSrc {
     inherit system;
-    overlays = [ cargo2nixOverlay ];
   };
   kaniko = (import ./nix/kaniko.nix) pkgs;
   manifest-tool = (import ./nix/manifest-tool.nix) pkgs;
   winscp = (import ./nix/winscp.nix) pkgs;
+in
+{
+  # --- Dev shell inherited from flake.nix ---
+  devShell = devShells.default;
 
-in {
+  # --- Continuous integration shell ---
-  # --- Rust Shell ---
+  # The shell used for all CI jobs (along with devShell)
-  # Use it to compile Garage
+  ci = pkgs.mkShell {
-  rust = pkgs.mkShell {
     nativeBuildInputs = with pkgs; [
-      #rustPlatform.rust.rustc
+      kaniko
-      rustPlatform.rust.cargo
+      manifest-tool
-      clang
-      mold
-      #clippy
-      rustfmt
-      #perl
-      #protobuf
-      #pkg-config
-      #openssl
-      file
-      #cargo2nix.packages.x86_64-linux.cargo2nix
-    ];
-  };
-
-  # --- Integration shell ---
-  # Use it to test Garage with common S3 clients
-  integration = pkgs.mkShell {
-    nativeBuildInputs = [
       winscp
-      pkgs.s3cmd
+
-      pkgs.awscli2
+      awscli2
-      pkgs.minio-client
+      file
-      pkgs.rclone
+      s3cmd
-      pkgs.socat
+      minio-client
-      pkgs.psmisc
+      rclone
-      pkgs.which
+      socat
-      pkgs.openssl
+      psmisc
-      pkgs.curl
+      which
-      pkgs.jq
+      openssl
+      curl
+      jq
     ];
-  };
-
-  # --- Release shell ---
-  # A shell built to make releasing easier
-  release = pkgs.mkShell {
     shellHook = ''
-      function refresh_toolchain {
-        pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
-        nix copy \
-          --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
-          $(nix-store -qR \
-            $(nix-build --no-build-output --no-out-link nix/toolchain.nix))
-        rm /tmp/nix-signing-key.sec
-      }
-
-      function refresh_cache {
-        pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
-        for attr in clippy.amd64 test.amd64 pkgs.{amd64,i386,arm,arm64}.{debug,release}; do
-          echo "Updating cache for ''${attr}"
-          derivation=$(nix-instantiate --attr ''${attr})
-          nix copy -j8 \
-            --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
-            $(nix-store -qR ''${derivation%\!bin})
-        done
-        rm /tmp/nix-signing-key.sec
-      }
-
-      function refresh_flake_cache {
-        pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
-        for attr in packages.x86_64-linux.default devShell.x86_64-linux; do
-          echo "Updating cache for ''${attr}"
-          derivation=$(nix path-info --derivation ".#''${attr}")
-          nix copy -j8 \
-            --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
-            $(nix-store -qR ''${derivation})
-        done
-        rm /tmp/nix-signing-key.sec
-      }
-
       function to_s3 {
         aws \
             --endpoint-url https://garage.deuxfleurs.fr \
@@ -167,7 +116,45 @@ in {
             s3://garagehq.deuxfleurs.fr/
       }
     '';
-    nativeBuildInputs = [ pkgs.awscli2 kaniko manifest-tool ];
+
+  };
+
+  # --- Cache shell ---
+  # A shell for refreshing caches
+  cache = pkgs.mkShell {
+    shellHook = ''
+      function refresh_toolchain {
+        pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
+        nix copy -j8 \
+          --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
+          $(nix-store -qR \
+              $(nix-build -j8 --no-build-output --no-out-link nix/toolchain.nix))
+        rm /tmp/nix-signing-key.sec
+      }
+
+      function refresh_cache {
+        pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
+        for attr in clippy.amd64 test.amd64 pkgs.{amd64,i386,arm,arm64}.release; do
+          echo "Updating cache for ''${attr}"
+          nix copy -j8 \
+            --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
+            $(nix path-info ''${attr} --file default.nix --derivation --recursive | sed 's/\.drv$/.drv^*/')
+
+        done
+        rm /tmp/nix-signing-key.sec
+      }
+
+      function refresh_flake_cache {
+        pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
+        for attr in packages.x86_64-linux.default devShells.x86_64-linux.default; do
+          echo "Updating cache for ''${attr}"
+          nix copy -j8 \
+            --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
+            ".#''${attr}"
+        done
+        rm /tmp/nix-signing-key.sec
+      }
+    '';
   };
 }