garage/default.nix

96 lines
3.1 KiB
Nix
Raw Normal View History

2021-10-04 16:27:57 +00:00
{
system ? builtins.currentSystem,
release ? false,
target ? "x86_64-unknown-linux-musl",
compileMode ? null,
git_version ? null,
}:
with import ./nix/common.nix;
let
crossSystem = { config = target; };
in let
pkgs = import pkgsSrc {
inherit system crossSystem;
overlays = [ cargo2nixOverlay ];
2021-10-04 16:27:57 +00:00
};
/*
2022-02-03 17:04:43 +00:00
Cargo2nix is built for rustOverlay which installs Rust from Mozilla releases.
We want our own Rust to avoir incompatibilities, like we had with musl 1.2.0.
rustc was built with musl < 1.2.0 and nix shipped musl >= 1.2.0 which lead to compilation breakage.
So we want a Rust release that is bound to our Nix repository to avoid these problems.
See here for more info: https://musl.libc.org/time64.html
Because Cargo2nix does not support the Rust environment shipped by NixOS,
we emulate the structure of the Rust object created by rustOverlay.
In practise, rustOverlay ships rustc+cargo in a single derivation while
NixOS ships them in separate ones. We reunite them with symlinkJoin.
2021-10-04 16:27:57 +00:00
*/
2022-02-03 17:04:43 +00:00
rustChannel = pkgs.symlinkJoin {
name ="rust-channel";
paths = [
pkgs.rustPlatform.rust.rustc
pkgs.rustPlatform.rust.cargo
];
};
2021-10-04 16:27:57 +00:00
2022-02-03 17:04:43 +00:00
overrides = pkgs.rustBuilder.overrides.all ++ [
Implement ListMultipartUploads (#171) Implement ListMultipartUploads, also refactor ListObjects and ListObjectsV2. It took me some times as I wanted to propose the following things: - Using an iterator instead of the loop+goto pattern. I find it easier to read and it should enable some optimizations. For example, when consuming keys of a common prefix, we do many [redundant checks](https://git.deuxfleurs.fr/Deuxfleurs/garage/src/branch/main/src/api/s3_list.rs#L125-L156) while the only thing to do is to [check if the following key is still part of the common prefix](https://git.deuxfleurs.fr/Deuxfleurs/garage/src/branch/feature/s3-multipart-compat/src/api/s3_list.rs#L476). - Try to name things (see ExtractionResult and RangeBegin enums) and to separate concerns (see ListQuery and Accumulator) - An IO closure to make unit tests possibles. - Unit tests, to track regressions and document how to interact with the code - Integration tests with `s3api`. In the future, I would like to move them in Rust with the aws rust SDK. Merging of the logic of ListMultipartUploads and ListObjects was not a goal but a consequence of the previous modifications. Some points that we might want to discuss: - ListObjectsV1, when using pagination and delimiters, has a weird behavior (it lists multiple times the same prefix) with `aws s3api` due to the fact that it can not use our optimization to skip the whole prefix. It is independant from my refactor and can be tested with the commented `s3api` tests in `test-smoke.sh`. It probably has the same weird behavior on the official AWS S3 implementation. - Considering ListMultipartUploads, I had to "abuse" upload id marker to support prefix skipping. I send an `upload-id-marker` with the hardcoded value `include` to emulate your "including" token. - Some ways to test ListMultipartUploads with existing software (my tests are limited to s3api for now). Co-authored-by: Quentin Dufour <quentin@deuxfleurs.fr> Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/171 Co-authored-by: Quentin <quentin@dufour.io> Co-committed-by: Quentin <quentin@dufour.io>
2022-01-12 18:04:55 +00:00
/*
We want to inject the git version while keeping the build deterministic.
As we do not want to consider the .git folder as part of the input source,
we ask the user (the CI often) to pass the value to Nix.
*/
2021-10-04 16:27:57 +00:00
(pkgs.rustBuilder.rustLib.makeOverride {
name = "garage";
overrideAttrs = drv: if git_version != null then {
preConfigure = ''
${drv.preConfigure or ""}
export GIT_VERSION="${git_version}"
'';
} else {};
})
Implement ListMultipartUploads (#171) Implement ListMultipartUploads, also refactor ListObjects and ListObjectsV2. It took me some times as I wanted to propose the following things: - Using an iterator instead of the loop+goto pattern. I find it easier to read and it should enable some optimizations. For example, when consuming keys of a common prefix, we do many [redundant checks](https://git.deuxfleurs.fr/Deuxfleurs/garage/src/branch/main/src/api/s3_list.rs#L125-L156) while the only thing to do is to [check if the following key is still part of the common prefix](https://git.deuxfleurs.fr/Deuxfleurs/garage/src/branch/feature/s3-multipart-compat/src/api/s3_list.rs#L476). - Try to name things (see ExtractionResult and RangeBegin enums) and to separate concerns (see ListQuery and Accumulator) - An IO closure to make unit tests possibles. - Unit tests, to track regressions and document how to interact with the code - Integration tests with `s3api`. In the future, I would like to move them in Rust with the aws rust SDK. Merging of the logic of ListMultipartUploads and ListObjects was not a goal but a consequence of the previous modifications. Some points that we might want to discuss: - ListObjectsV1, when using pagination and delimiters, has a weird behavior (it lists multiple times the same prefix) with `aws s3api` due to the fact that it can not use our optimization to skip the whole prefix. It is independant from my refactor and can be tested with the commented `s3api` tests in `test-smoke.sh`. It probably has the same weird behavior on the official AWS S3 implementation. - Considering ListMultipartUploads, I had to "abuse" upload id marker to support prefix skipping. I send an `upload-id-marker` with the hardcoded value `include` to emulate your "including" token. - Some ways to test ListMultipartUploads with existing software (my tests are limited to s3api for now). Co-authored-by: Quentin Dufour <quentin@deuxfleurs.fr> Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/171 Co-authored-by: Quentin <quentin@dufour.io> Co-committed-by: Quentin <quentin@dufour.io>
2022-01-12 18:04:55 +00:00
/*
On a sandbox pure NixOS environment, /usr/bin/file is not available.
This is a known problem: https://github.com/NixOS/nixpkgs/issues/98440
We simply patch the file as suggested
*/
/*(pkgs.rustBuilder.rustLib.makeOverride {
name = "libsodium-sys";
overrideAttrs = drv: {
preConfigure = ''
${drv.preConfigure or ""}
sed -i 's,/usr/bin/file,${file}/bin/file,g' ./configure
'';
}
})*/
2021-10-04 16:27:57 +00:00
];
packageFun = import ./Cargo.nix;
2022-02-03 17:04:43 +00:00
/*
The following definition is not elegant as we use a low level function of Cargo2nix
that enables us to pass our custom rustChannel object
*/
2021-10-04 16:27:57 +00:00
rustPkgs = pkgs.rustBuilder.makePackageSet {
inherit packageFun rustChannel release;
packageOverrides = overrides;
2022-02-03 17:04:43 +00:00
target = null; /* we set target to null because we want that cargo2nix computes it automatically */
2021-10-04 16:27:57 +00:00
buildRustPackages = pkgs.buildPackages.rustBuilder.makePackageSet {
inherit rustChannel packageFun;
packageOverrides = overrides;
2022-02-03 17:04:43 +00:00
target = null; /* we set target to null because we want that cargo2nix computes it automatically */
2021-10-04 16:27:57 +00:00
};
};
2022-02-03 17:04:43 +00:00
2021-10-04 16:27:57 +00:00
in
if compileMode == "test"
2022-02-03 17:04:43 +00:00
then pkgs.symlinkJoin {
name ="garage-tests";
paths = builtins.map (key: rustPkgs.workspace.${key} { inherit compileMode; }) (builtins.attrNames rustPkgs.workspace);
}
2021-10-04 16:27:57 +00:00
else rustPkgs.workspace.garage { inherit compileMode; }