Add integration tests to Drone

2022-02-03 18:04:43 +01:00 · 2022-02-03 18:04:43 +01:00 · 5d19f3d2d7
parent 084dcdbd3a
commit 5d19f3d2d7
6 changed files with 58 additions and 35 deletions
--- a/.drone.yml
+++ b/.drone.yml
@ -48,8 +48,10 @@ steps:
    commands:
      - nix-build --no-build-output --option log-lines 100 --argstr target x86_64-unknown-linux-musl --arg release false --argstr git_version $DRONE_COMMIT

-  - name: unit tests
+  - name: unit + func tests
    image: nixpkgs/nix:nixos-21.05
+    environment:
+      GARAGE_TEST_INTEGRATION_EXE: result/bin/garage
    volumes:
    - name: nix_store
      path: /nix
@ -62,13 +64,14 @@ steps:
          --option log-lines 100 \
          --argstr target x86_64-unknown-linux-musl \
          --argstr compileMode test
-      - ./result*/bin/garage_api*
-      - ./result*/bin/garage_model*
-      - ./result*/bin/garage_rpc*
-      - ./result*/bin/garage_table*
-      - ./result*/bin/garage_util*
-      - ./result*/bin/garage_web*
-      - ./result*/bin/garage*
+      - ./result/bin/garage_api-*
+      - ./result/bin/garage_model-*
+      - ./result/bin/garage_rpc-*
+      - ./result/bin/garage_table-*
+      - ./result/bin/garage_util-*
+      - ./result/bin/garage_web-*
+      - ./result/bin/garage-*
+      - ./result/bin/integration-*

  - name: smoke-test
    image: nixpkgs/nix:nixos-21.05
@ -470,6 +473,6 @@ node:

 ---
 kind: signature
-hmac: 928ea1bb59f3ac19b5ddd2a184f17b7c728cc355877c34e61b3d1b421544d4c3
+hmac: 3fc19d6f9a3555519c8405e3281b2e74289bb802f644740d5481d53df3a01fa4

 ...
--- a/Cargo.nix
+++ b/Cargo.nix
@ -1,4 +1,4 @@
-# This file was @generated by cargo2nix 0.9.0.
+# This file was @generated by cargo2nix 0.10.0.
 # It is not intended to be manually edited.

 args@{
@ -17,6 +17,9 @@ args@{
  hostPlatform,
  hostPlatformCpu ? null,
  hostPlatformFeatures ? [],
+  target ? null,
+  codegenOpts ? null,
+  profileOpts ? null,
  mkRustCrate,
  rustLib,
  lib,
@ -33,12 +36,12 @@ in let
  rootFeatures' = expandFeatures rootFeatures;
  overridableMkRustCrate = f:
    let
-      drvs = genDrvsByProfile profilesByName ({ profile, profileName }: mkRustCrate ({ inherit release profile hostPlatformCpu hostPlatformFeatures; } // (f profileName)));
+      drvs = genDrvsByProfile profilesByName ({ profile, profileName }: mkRustCrate ({ inherit release profile hostPlatformCpu hostPlatformFeatures target profileOpts codegenOpts; } // (f profileName)));
    in { compileMode ? null, profileName ? decideProfile compileMode release }:
      let drv = drvs.${profileName}; in if compileMode == null then drv else drv.override { inherit compileMode; };
 in
 {
-  cargo2nixVersion = "0.9.0";
+  cargo2nixVersion = "0.10.0";
  workspace = {
    garage_util = rustPackages.unknown.garage_util."0.6.0";
    garage_rpc = rustPackages.unknown.garage_rpc."0.6.0";
@ -2283,7 +2286,7 @@ in
    ];
    dependencies = {
      ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.115" { inherit profileName; };
-      ${ if hostPlatform.parsed.kernel.name == "dragonfly" || hostPlatform.parsed.kernel.name == "freebsd" || hostPlatform.parsed.kernel.name == "illumos" || hostPlatform.parsed.kernel.name == "netbsd" || hostPlatform.parsed.kernel.name == "openbsd" || hostPlatform.parsed.kernel.name == "solaris" || hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "once_cell" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.8.0" { inherit profileName; };
+      ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "dragonfly" || hostPlatform.parsed.kernel.name == "freebsd" || hostPlatform.parsed.kernel.name == "illumos" || hostPlatform.parsed.kernel.name == "netbsd" || hostPlatform.parsed.kernel.name == "openbsd" || hostPlatform.parsed.kernel.name == "solaris" then "once_cell" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.8.0" { inherit profileName; };
      ${ if hostPlatform.parsed.cpu.name == "i686" || hostPlatform.parsed.cpu.name == "x86_64" || (hostPlatform.parsed.cpu.name == "aarch64" || hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l") && (hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "fuchsia" || hostPlatform.parsed.kernel.name == "linux") then "spin" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".spin."0.5.2" { inherit profileName; };
      untrusted = rustPackages."registry+https://github.com/rust-lang/crates.io-index".untrusted."0.7.1" { inherit profileName; };
      ${ if hostPlatform.parsed.cpu.name == "wasm32" && hostPlatform.parsed.vendor.name == "unknown" && hostPlatform.parsed.kernel.name == "unknown" && hostPlatform.parsed.abi.name == "" then "web_sys" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".web-sys."0.3.56" { inherit profileName; };
--- a/default.nix
+++ b/default.nix
@ -17,13 +17,25 @@ in let
  };

  /*
-   The following complexity should be abstracted by makePackageSet' (note the final quote).
-   However its code uses deprecated features of rust-overlay that can lead to bug.
-   Instead, we build our own rustChannel object with the recommended API of rust-overlay.
+   Cargo2nix is built for rustOverlay which installs Rust from Mozilla releases.
+   We want our own Rust to avoir incompatibilities, like we had with musl 1.2.0.
+   rustc was built with musl < 1.2.0 and nix shipped musl >= 1.2.0 which lead to compilation breakage.
+   So we want a Rust release that is bound to our Nix repository to avoid these problems.
+   See here for more info: https://musl.libc.org/time64.html
+   Because Cargo2nix does not support the Rust environment shipped by NixOS,
+   we emulate the structure of the Rust object created by rustOverlay.
+   In practise, rustOverlay ships rustc+cargo in a single derivation while
+   NixOS ships them in separate ones. We reunite them with symlinkJoin.
   */
-  rustChannel = pkgs.rustPlatform.rust;
+  rustChannel = pkgs.symlinkJoin {
+    name ="rust-channel";
+    paths = [ 
+      pkgs.rustPlatform.rust.rustc
+      pkgs.rustPlatform.rust.cargo
+    ];
+  };

-  overrides = pkgs.buildPackages.rustBuilder.overrides.all ++ [
+  overrides = pkgs.rustBuilder.overrides.all ++ [
    /*
     We want to inject the git version while keeping the build deterministic.
     As we do not want to consider the .git folder as part of the input source,
@ -57,30 +69,27 @@ in let

  packageFun = import ./Cargo.nix;

+  /*
+   The following definition is not elegant as we use a low level function of Cargo2nix
+   that enables us to pass our custom rustChannel object
+  */ 
  rustPkgs = pkgs.rustBuilder.makePackageSet {
    inherit packageFun rustChannel release;
    packageOverrides = overrides;
+    target = null; /* we set target to null because we want that cargo2nix computes it automatically */

    buildRustPackages = pkgs.buildPackages.rustBuilder.makePackageSet {
      inherit rustChannel packageFun;
      packageOverrides = overrides;
+      target = null; /* we set target to null because we want that cargo2nix computes it automatically */ 
    };
-
-    localPatterns = [
-      /*
-       The way the default rules are written make think we match recursively, on full path, but the rules are misleading.
-       In fact, the regex is only called on root elements of the crate (and not recursively).
-       This behavior does not work well with our nested modules.
-       We tried to build a "deny list" but negative lookup ahead are not supported on Nix.
-       As a workaround, we have to register all our submodules in this allow list...
-       */
-       ''^(src|tests)''          # fixed default
-       ''.*\.(rs|toml)$''        # fixed default
-       ''^(crdt|replication|cli|helper|signature|common|ext)''   # our crate submodules
-    ];
  };

+
 in
  if compileMode == "test"
-    then builtins.mapAttrs (name: value: rustPkgs.workspace.${name} { inherit compileMode; }) rustPkgs.workspace
+    then pkgs.symlinkJoin {
+      name ="garage-tests";
+      paths = builtins.map (key: rustPkgs.workspace.${key} { inherit compileMode; }) (builtins.attrNames rustPkgs.workspace);
+    }
    else rustPkgs.workspace.garage { inherit compileMode; }
--- a/nix/common.nix
+++ b/nix/common.nix
@ -8,12 +8,14 @@ rec {
    sha256 = "1xy9zpypqfxs5gcq5dcla4bfkhxmh5nzn9dyqkr03lqycm9wg5cr";
  };
  cargo2nixSrc = fetchGit {
-    # As of 2021-10-06
+    # As of 2022-02-03
    url = "https://github.com/superboum/cargo2nix";
    ref = "backward-compat";
    rev = "08d963f32a774353ee8acf3f61749915875c1ec4";
  };

+
+
  /*
   * Shared objects
   */
--- a/shell.nix
+++ b/shell.nix
@ -76,7 +76,7 @@ function refresh_toolchain {
     pkgs.rustPlatform.rust.cargo
     pkgs.clippy
     pkgs.rustfmt
-     /* cargo2nix.packages.x86_64-linux.cargo2nix */
+     cargo2nix.packages.x86_64-linux.cargo2nix
    ] else [])
   ++
   (if integration then [ 
--- a/src/garage/tests/common/garage.rs
+++ b/src/garage/tests/common/garage.rs
@ -1,3 +1,4 @@
+use std::env::var_os;
 use std::mem::MaybeUninit;
 use std::path::{Path, PathBuf};
 use std::process;
@ -203,7 +204,12 @@ pub fn instance() -> &'static Instance {
 }

 pub fn command(config_path: &Path) -> process::Command {
-	let mut command = process::Command::new(env!("CARGO_BIN_EXE_garage"));
+	let mut command = process::Command::new(
+		var_os("GARAGE_TEST_INTEGRATION_EXE")
+			.as_ref()
+			.and_then(|e| e.to_str())
+			.unwrap_or(env!("CARGO_BIN_EXE_garage")),
+	);

 	command.arg("-c").arg(config_path);