Add integration tests to Drone

.drone.yml

@@ -48,8 +48,10 @@ steps:
     commands:
       - nix-build --no-build-output --option log-lines 100 --argstr target x86_64-unknown-linux-musl --arg release false --argstr git_version $DRONE_COMMIT
 
-  - name: unit tests
+  - name: unit + func tests
     image: nixpkgs/nix:nixos-21.05
+    environment:
+      GARAGE_TEST_INTEGRATION_EXE: result/bin/garage
     volumes:
     - name: nix_store
       path: /nix
@@ -62,13 +64,14 @@ steps:
           --option log-lines 100 \
           --argstr target x86_64-unknown-linux-musl \
           --argstr compileMode test
-      - ./result*/bin/garage_api*
+      - ./result/bin/garage_api-*
-      - ./result*/bin/garage_model*
+      - ./result/bin/garage_model-*
-      - ./result*/bin/garage_rpc*
+      - ./result/bin/garage_rpc-*
-      - ./result*/bin/garage_table*
+      - ./result/bin/garage_table-*
-      - ./result*/bin/garage_util*
+      - ./result/bin/garage_util-*
-      - ./result*/bin/garage_web*
+      - ./result/bin/garage_web-*
-      - ./result*/bin/garage*
+      - ./result/bin/garage-*
+      - ./result/bin/integration-*
 
   - name: smoke-test
     image: nixpkgs/nix:nixos-21.05
@@ -512,6 +515,6 @@ node:
 
 ---
 kind: signature
-hmac: 1c33490cc2902564c4250a409c156683d0d549b8c9d5aee4e46d1bde4e0ccf2c
+hmac: 0ba1f5febd521c77c4c0ecb6724888a8d3307024fc74feea1d5bf6bb3bce8429
 
 ...

Cargo.nix

@@ -1,4 +1,4 @@
-# This file was @generated by cargo2nix 0.9.0.
+# This file was @generated by cargo2nix 0.10.0.
 # It is not intended to be manually edited.
 
 args@{
@@ -17,6 +17,9 @@ args@{
   hostPlatform,
   hostPlatformCpu ? null,
   hostPlatformFeatures ? [],
+  target ? null,
+  codegenOpts ? null,
+  profileOpts ? null,
   mkRustCrate,
   rustLib,
   lib,
@@ -33,12 +36,12 @@ in let
   rootFeatures' = expandFeatures rootFeatures;
   overridableMkRustCrate = f:
     let
-      drvs = genDrvsByProfile profilesByName ({ profile, profileName }: mkRustCrate ({ inherit release profile hostPlatformCpu hostPlatformFeatures; } // (f profileName)));
+      drvs = genDrvsByProfile profilesByName ({ profile, profileName }: mkRustCrate ({ inherit release profile hostPlatformCpu hostPlatformFeatures target profileOpts codegenOpts; } // (f profileName)));
     in { compileMode ? null, profileName ? decideProfile compileMode release }:
       let drv = drvs.${profileName}; in if compileMode == null then drv else drv.override { inherit compileMode; };
 in
 {
-  cargo2nixVersion = "0.9.0";
+  cargo2nixVersion = "0.10.0";
   workspace = {
     garage_util = rustPackages.unknown.garage_util."0.6.0";
     garage_rpc = rustPackages.unknown.garage_rpc."0.6.0";
@@ -2283,7 +2286,7 @@ in
     ];
     dependencies = {
       ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.115" { inherit profileName; };
-      ${ if hostPlatform.parsed.kernel.name == "dragonfly" || hostPlatform.parsed.kernel.name == "freebsd" || hostPlatform.parsed.kernel.name == "illumos" || hostPlatform.parsed.kernel.name == "netbsd" || hostPlatform.parsed.kernel.name == "openbsd" || hostPlatform.parsed.kernel.name == "solaris" || hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "once_cell" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.8.0" { inherit profileName; };
+      ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "dragonfly" || hostPlatform.parsed.kernel.name == "freebsd" || hostPlatform.parsed.kernel.name == "illumos" || hostPlatform.parsed.kernel.name == "netbsd" || hostPlatform.parsed.kernel.name == "openbsd" || hostPlatform.parsed.kernel.name == "solaris" then "once_cell" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.8.0" { inherit profileName; };
       ${ if hostPlatform.parsed.cpu.name == "i686" || hostPlatform.parsed.cpu.name == "x86_64" || (hostPlatform.parsed.cpu.name == "aarch64" || hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l") && (hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "fuchsia" || hostPlatform.parsed.kernel.name == "linux") then "spin" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".spin."0.5.2" { inherit profileName; };
       untrusted = rustPackages."registry+https://github.com/rust-lang/crates.io-index".untrusted."0.7.1" { inherit profileName; };
       ${ if hostPlatform.parsed.cpu.name == "wasm32" && hostPlatform.parsed.vendor.name == "unknown" && hostPlatform.parsed.kernel.name == "unknown" && hostPlatform.parsed.abi.name == "" then "web_sys" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".web-sys."0.3.56" { inherit profileName; };

default.nix

@@ -17,13 +17,25 @@ in let
   };
 
   /*
-   The following complexity should be abstracted by makePackageSet' (note the final quote).
+   Cargo2nix is built for rustOverlay which installs Rust from Mozilla releases.
-   However its code uses deprecated features of rust-overlay that can lead to bug.
+   We want our own Rust to avoir incompatibilities, like we had with musl 1.2.0.
-   Instead, we build our own rustChannel object with the recommended API of rust-overlay.
+   rustc was built with musl < 1.2.0 and nix shipped musl >= 1.2.0 which lead to compilation breakage.
+   So we want a Rust release that is bound to our Nix repository to avoid these problems.
+   See here for more info: https://musl.libc.org/time64.html
+   Because Cargo2nix does not support the Rust environment shipped by NixOS,
+   we emulate the structure of the Rust object created by rustOverlay.
+   In practise, rustOverlay ships rustc+cargo in a single derivation while
+   NixOS ships them in separate ones. We reunite them with symlinkJoin.
    */
-  rustChannel = pkgs.rustPlatform.rust;
+  rustChannel = pkgs.symlinkJoin {
+    name ="rust-channel";
+    paths = [ 
+      pkgs.rustPlatform.rust.rustc
+      pkgs.rustPlatform.rust.cargo
+    ];
+  };
 
-  overrides = pkgs.buildPackages.rustBuilder.overrides.all ++ [
+  overrides = pkgs.rustBuilder.overrides.all ++ [
     /*
      We want to inject the git version while keeping the build deterministic.
      As we do not want to consider the .git folder as part of the input source,
@@ -57,30 +69,27 @@ in let
 
   packageFun = import ./Cargo.nix;
 
+  /*
+   The following definition is not elegant as we use a low level function of Cargo2nix
+   that enables us to pass our custom rustChannel object
+  */ 
   rustPkgs = pkgs.rustBuilder.makePackageSet {
     inherit packageFun rustChannel release;
     packageOverrides = overrides;
+    target = null; /* we set target to null because we want that cargo2nix computes it automatically */
 
     buildRustPackages = pkgs.buildPackages.rustBuilder.makePackageSet {
       inherit rustChannel packageFun;
       packageOverrides = overrides;
+      target = null; /* we set target to null because we want that cargo2nix computes it automatically */ 
+    };
   };
 
-    localPatterns = [
-      /*
-       The way the default rules are written make think we match recursively, on full path, but the rules are misleading.
-       In fact, the regex is only called on root elements of the crate (and not recursively).
-       This behavior does not work well with our nested modules.
-       We tried to build a "deny list" but negative lookup ahead are not supported on Nix.
-       As a workaround, we have to register all our submodules in this allow list...
-       */
-       ''^(src|tests)''          # fixed default
-       ''.*\.(rs|toml)$''        # fixed default
-       ''^(crdt|replication|cli|helper|signature|common|ext)''   # our crate submodules
-    ];
-  };
 
 in
   if compileMode == "test"
-    then builtins.mapAttrs (name: value: rustPkgs.workspace.${name} { inherit compileMode; }) rustPkgs.workspace
+    then pkgs.symlinkJoin {
+      name ="garage-tests";
+      paths = builtins.map (key: rustPkgs.workspace.${key} { inherit compileMode; }) (builtins.attrNames rustPkgs.workspace);
+    }
     else rustPkgs.workspace.garage { inherit compileMode; }

nix/common.nix

@@ -8,11 +8,14 @@ rec {
     sha256 = "1xy9zpypqfxs5gcq5dcla4bfkhxmh5nzn9dyqkr03lqycm9wg5cr";
   };
   cargo2nixSrc = fetchGit {
-    # As of 2021-10-06
+    # As of 2022-02-03
     url = "https://github.com/superboum/cargo2nix";
-    rev = "1364752cd784764db2ef5b1e1248727cebfae2ce";
+    ref = "backward-compat";
+    rev = "08d963f32a774353ee8acf3f61749915875c1ec4";
   };
 
+
+
   /*
    * Shared objects
    */

shell.nix

@@ -70,7 +70,7 @@ function refresh_toolchain {
      pkgs.rustPlatform.rust.cargo
      pkgs.clippy
      pkgs.rustfmt
-     /*(pkgs.callPackage cargo2nix {}).package*/
+     cargo2nix.packages.x86_64-linux.cargo2nix
     ] else [])
    ++
    (if integration then [ 

src/garage/tests/common/garage.rs

@@ -1,3 +1,4 @@
+use std::env::var_os;
 use std::mem::MaybeUninit;
 use std::path::{Path, PathBuf};
 use std::process;
@@ -203,7 +204,12 @@ pub fn instance() -> &'static Instance {
 }
 
 pub fn command(config_path: &Path) -> process::Command {
-	let mut command = process::Command::new(env!("CARGO_BIN_EXE_garage"));
+	let mut command = process::Command::new(
+		var_os("GARAGE_TEST_INTEGRATION_EXE")
+			.as_ref()
+			.and_then(|e| e.to_str())
+			.unwrap_or(env!("CARGO_BIN_EXE_garage")),
+	);
 
 	command.arg("-c").arg(config_path);