From 4d30e62db456097563c574b9dfd22b138d700087 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 16 Dec 2021 13:17:09 +0100 Subject: [PATCH] New buckets for 0.6.0: migration code and build files --- Cargo.lock | 31 +++--- Cargo.nix | 200 ++++++++++++++++++++++++++++--------- src/garage/admin.rs | 19 ++++ src/garage/cli/cmd.rs | 3 + src/garage/cli/structs.rs | 22 ++++ src/model/Cargo.toml | 2 +- src/model/key_table.rs | 27 +++++ src/model/lib.rs | 1 + src/model/migrate.rs | 93 +++++++++++++++++ src/model/object_table.rs | 71 +++++++++++++ src/model/version_table.rs | 35 +++++++ 11 files changed, 441 insertions(+), 63 deletions(-) create mode 100644 src/model/migrate.rs diff --git a/Cargo.lock b/Cargo.lock index fecfaea6..8630855f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -444,17 +444,17 @@ dependencies = [ [[package]] name = "garage_model" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56150ee02bc26c77996b19fee0851f7d53cf42ae80370a8cf3a5dd5bb0bba76" +checksum = "584619e8999713d73761775591ad6f01ff8c9d724f3b20984f5932f1fc7f9988" dependencies = [ "arc-swap", "async-trait", "futures", "futures-util", - "garage_rpc 0.5.0", - "garage_table 0.5.0", - "garage_util 0.5.0", + "garage_rpc 0.5.1", + "garage_table 0.5.1", + "garage_util 0.5.1", "hex", "log", "netapp", @@ -464,6 +464,7 @@ dependencies = [ "serde_bytes", "sled", "tokio", + "zstd", ] [[package]] @@ -474,7 +475,7 @@ dependencies = [ "async-trait", "futures", "futures-util", - "garage_model 0.5.0", + "garage_model 0.5.1", "garage_rpc 0.6.0", "garage_table 0.6.0", "garage_util 0.6.0", @@ -492,16 +493,16 @@ dependencies = [ [[package]] name = "garage_rpc" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5743c49f616b260f548454ff52b81d10372593d4c4bc01d516ee3c3c4e515a" +checksum = "81e693aa4582cfe7a7ce70c07880e3662544b5d0cd68bc4b59c53febfbb8d1ec" dependencies = [ "arc-swap", "async-trait", "bytes 1.1.0", "futures", "futures-util", - "garage_util 0.5.0", + "garage_util 0.5.1", "gethostname", "hex", "hyper", @@ -544,16 +545,16 @@ dependencies = [ [[package]] name = "garage_table" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "378ffd69e8fd084e0817dc64a23a1692b58ffc86509ac2cadc64aa2d83c3e1e0" +checksum = "5c3557f3757e2acd29eaee86804d4e6c38d2abda81b4b349d8a0d2277044265c" dependencies = [ "async-trait", "bytes 1.1.0", "futures", "futures-util", - "garage_rpc 0.5.0", - "garage_util 0.5.0", + "garage_rpc 0.5.1", + "garage_util 0.5.1", "hexdump", "log", "rand", @@ -586,9 +587,9 @@ dependencies = [ [[package]] name = "garage_util" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5282e613b4da5ecca5bfec8c48ce9f25226cc1f35fbc439ed5fc698cce1aa549" +checksum = "1e096994382447431e2f3c70e3685eb8b24c00eceff8667bb22a2a27ff17832f" dependencies = [ "blake2", "chrono", diff --git a/Cargo.nix b/Cargo.nix index 5b38c55e..78f2b5c0 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -40,13 +40,13 @@ in { cargo2nixVersion = "0.9.0"; workspace = { - garage_util = rustPackages.unknown.garage_util."0.5.0"; - garage_rpc = rustPackages.unknown.garage_rpc."0.5.0"; - garage_table = rustPackages.unknown.garage_table."0.5.0"; - garage_model = rustPackages.unknown.garage_model."0.5.0"; - garage_api = rustPackages.unknown.garage_api."0.5.0"; - garage_web = rustPackages.unknown.garage_web."0.5.0"; - garage = rustPackages.unknown.garage."0.5.0"; + garage_util = rustPackages.unknown.garage_util."0.6.0"; + garage_rpc = rustPackages.unknown.garage_rpc."0.6.0"; + garage_table = rustPackages.unknown.garage_table."0.6.0"; + garage_model = rustPackages.unknown.garage_model."0.6.0"; + garage_api = rustPackages.unknown.garage_api."0.6.0"; + garage_web = rustPackages.unknown.garage_web."0.6.0"; + garage = rustPackages.unknown.garage."0.6.0"; }; "registry+https://github.com/rust-lang/crates.io-index".aho-corasick."0.7.18" = overridableMkRustCrate (profileName: rec { name = "aho-corasick"; @@ -253,7 +253,7 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469"; }; dependencies = { - ${ if hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" || hostPlatform.config == "aarch64-apple-darwin" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.103" { inherit profileName; }; + ${ if hostPlatform.config == "aarch64-apple-darwin" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.103" { inherit profileName; }; }; }); @@ -613,9 +613,9 @@ in }; }); - "unknown".garage."0.5.0" = overridableMkRustCrate (profileName: rec { + "unknown".garage."0.6.0" = overridableMkRustCrate (profileName: rec { name = "garage"; - version = "0.5.0"; + version = "0.6.0"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/garage"); dependencies = { @@ -623,12 +623,12 @@ in bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.17" { inherit profileName; }; futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.17" { inherit profileName; }; - garage_api = rustPackages."unknown".garage_api."0.5.0" { inherit profileName; }; - garage_model = rustPackages."unknown".garage_model."0.5.0" { inherit profileName; }; - garage_rpc = rustPackages."unknown".garage_rpc."0.5.0" { inherit profileName; }; - garage_table = rustPackages."unknown".garage_table."0.5.0" { inherit profileName; }; - garage_util = rustPackages."unknown".garage_util."0.5.0" { inherit profileName; }; - garage_web = rustPackages."unknown".garage_web."0.5.0" { inherit profileName; }; + garage_api = rustPackages."unknown".garage_api."0.6.0" { inherit profileName; }; + garage_model = rustPackages."unknown".garage_model."0.6.0" { inherit profileName; }; + garage_rpc = rustPackages."unknown".garage_rpc."0.6.0" { inherit profileName; }; + garage_table = rustPackages."unknown".garage_table."0.6.0" { inherit profileName; }; + garage_util = rustPackages."unknown".garage_util."0.6.0" { inherit profileName; }; + garage_web = rustPackages."unknown".garage_web."0.6.0" { inherit profileName; }; git_version = rustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.5" { inherit profileName; }; hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }; sodiumoxide = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }; @@ -645,9 +645,9 @@ in }; }); - "unknown".garage_api."0.5.0" = overridableMkRustCrate (profileName: rec { + "unknown".garage_api."0.6.0" = overridableMkRustCrate (profileName: rec { name = "garage_api"; - version = "0.5.0"; + version = "0.6.0"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/api"); dependencies = { @@ -658,9 +658,9 @@ in err_derive = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.0" { profileName = "__noProfile"; }; futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.17" { inherit profileName; }; futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.17" { inherit profileName; }; - garage_model = rustPackages."unknown".garage_model."0.5.0" { inherit profileName; }; - garage_table = rustPackages."unknown".garage_table."0.5.0" { inherit profileName; }; - garage_util = rustPackages."unknown".garage_util."0.5.0" { inherit profileName; }; + garage_model = rustPackages."unknown".garage_model."0.6.0" { inherit profileName; }; + garage_table = rustPackages."unknown".garage_table."0.6.0" { inherit profileName; }; + garage_util = rustPackages."unknown".garage_util."0.6.0" { inherit profileName; }; hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }; hmac = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hmac."0.10.1" { inherit profileName; }; http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.5" { inherit profileName; }; @@ -674,25 +674,26 @@ in quick_xml = rustPackages."registry+https://github.com/rust-lang/crates.io-index".quick-xml."0.21.0" { inherit profileName; }; roxmltree = rustPackages."registry+https://github.com/rust-lang/crates.io-index".roxmltree."0.14.1" { inherit profileName; }; serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.130" { inherit profileName; }; + serde_bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }; sha2 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".sha2."0.9.8" { inherit profileName; }; tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.12.0" { inherit profileName; }; url = rustPackages."registry+https://github.com/rust-lang/crates.io-index".url."2.2.2" { inherit profileName; }; }; }); - "unknown".garage_model."0.5.0" = overridableMkRustCrate (profileName: rec { + "registry+https://github.com/rust-lang/crates.io-index".garage_model."0.5.1" = overridableMkRustCrate (profileName: rec { name = "garage_model"; - version = "0.5.0"; - registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/model"); + version = "0.5.1"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "584619e8999713d73761775591ad6f01ff8c9d724f3b20984f5932f1fc7f9988"; }; dependencies = { arc_swap = rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.4.0" { inherit profileName; }; async_trait = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.51" { profileName = "__noProfile"; }; futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.17" { inherit profileName; }; futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.17" { inherit profileName; }; - garage_rpc = rustPackages."unknown".garage_rpc."0.5.0" { inherit profileName; }; - garage_table = rustPackages."unknown".garage_table."0.5.0" { inherit profileName; }; - garage_util = rustPackages."unknown".garage_util."0.5.0" { inherit profileName; }; + garage_rpc = rustPackages."registry+https://github.com/rust-lang/crates.io-index".garage_rpc."0.5.1" { inherit profileName; }; + garage_table = rustPackages."registry+https://github.com/rust-lang/crates.io-index".garage_table."0.5.1" { inherit profileName; }; + garage_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".garage_util."0.5.1" { inherit profileName; }; hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }; log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; }; netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.3.0" { inherit profileName; }; @@ -706,18 +707,45 @@ in }; }); - "unknown".garage_rpc."0.5.0" = overridableMkRustCrate (profileName: rec { - name = "garage_rpc"; - version = "0.5.0"; + "unknown".garage_model."0.6.0" = overridableMkRustCrate (profileName: rec { + name = "garage_model"; + version = "0.6.0"; registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/rpc"); + src = fetchCrateLocal (workspaceSrc + "/src/model"); + dependencies = { + arc_swap = rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.4.0" { inherit profileName; }; + async_trait = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.51" { profileName = "__noProfile"; }; + futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.17" { inherit profileName; }; + futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.17" { inherit profileName; }; + garage_model_050 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".garage_model."0.5.1" { inherit profileName; }; + garage_rpc = rustPackages."unknown".garage_rpc."0.6.0" { inherit profileName; }; + garage_table = rustPackages."unknown".garage_table."0.6.0" { inherit profileName; }; + garage_util = rustPackages."unknown".garage_util."0.6.0" { inherit profileName; }; + hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }; + log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; }; + netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.3.0" { inherit profileName; }; + rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.4" { inherit profileName; }; + rmp_serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }; + serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.130" { inherit profileName; }; + serde_bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }; + sled = rustPackages."registry+https://github.com/rust-lang/crates.io-index".sled."0.34.7" { inherit profileName; }; + tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.12.0" { inherit profileName; }; + zstd = rustPackages."registry+https://github.com/rust-lang/crates.io-index".zstd."0.9.0+zstd.1.5.0" { inherit profileName; }; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".garage_rpc."0.5.1" = overridableMkRustCrate (profileName: rec { + name = "garage_rpc"; + version = "0.5.1"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "81e693aa4582cfe7a7ce70c07880e3662544b5d0cd68bc4b59c53febfbb8d1ec"; }; dependencies = { arc_swap = rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.4.0" { inherit profileName; }; async_trait = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.51" { profileName = "__noProfile"; }; bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.17" { inherit profileName; }; futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.17" { inherit profileName; }; - garage_util = rustPackages."unknown".garage_util."0.5.0" { inherit profileName; }; + garage_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".garage_util."0.5.1" { inherit profileName; }; gethostname = rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.2.1" { inherit profileName; }; hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }; hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.13" { inherit profileName; }; @@ -734,18 +762,46 @@ in }; }); - "unknown".garage_table."0.5.0" = overridableMkRustCrate (profileName: rec { - name = "garage_table"; - version = "0.5.0"; + "unknown".garage_rpc."0.6.0" = overridableMkRustCrate (profileName: rec { + name = "garage_rpc"; + version = "0.6.0"; registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/table"); + src = fetchCrateLocal (workspaceSrc + "/src/rpc"); + dependencies = { + arc_swap = rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.4.0" { inherit profileName; }; + async_trait = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.51" { profileName = "__noProfile"; }; + bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; + futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.17" { inherit profileName; }; + futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.17" { inherit profileName; }; + garage_util = rustPackages."unknown".garage_util."0.6.0" { inherit profileName; }; + gethostname = rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.2.1" { inherit profileName; }; + hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }; + hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.13" { inherit profileName; }; + sodiumoxide = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }; + log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; }; + netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.3.0" { inherit profileName; }; + rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.4" { inherit profileName; }; + rmp_serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }; + serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.130" { inherit profileName; }; + serde_bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }; + serde_json = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.68" { inherit profileName; }; + tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.12.0" { inherit profileName; }; + tokio_stream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.7" { inherit profileName; }; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".garage_table."0.5.1" = overridableMkRustCrate (profileName: rec { + name = "garage_table"; + version = "0.5.1"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "5c3557f3757e2acd29eaee86804d4e6c38d2abda81b4b349d8a0d2277044265c"; }; dependencies = { async_trait = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.51" { profileName = "__noProfile"; }; bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.17" { inherit profileName; }; futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.17" { inherit profileName; }; - garage_rpc = rustPackages."unknown".garage_rpc."0.5.0" { inherit profileName; }; - garage_util = rustPackages."unknown".garage_util."0.5.0" { inherit profileName; }; + garage_rpc = rustPackages."registry+https://github.com/rust-lang/crates.io-index".garage_rpc."0.5.1" { inherit profileName; }; + garage_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".garage_util."0.5.1" { inherit profileName; }; hexdump = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }; log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; }; rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.4" { inherit profileName; }; @@ -757,9 +813,59 @@ in }; }); - "unknown".garage_util."0.5.0" = overridableMkRustCrate (profileName: rec { + "unknown".garage_table."0.6.0" = overridableMkRustCrate (profileName: rec { + name = "garage_table"; + version = "0.6.0"; + registry = "unknown"; + src = fetchCrateLocal (workspaceSrc + "/src/table"); + dependencies = { + async_trait = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.51" { profileName = "__noProfile"; }; + bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; + futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.17" { inherit profileName; }; + futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.17" { inherit profileName; }; + garage_rpc = rustPackages."unknown".garage_rpc."0.6.0" { inherit profileName; }; + garage_util = rustPackages."unknown".garage_util."0.6.0" { inherit profileName; }; + hexdump = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }; + log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; }; + rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.4" { inherit profileName; }; + rmp_serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }; + serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.130" { inherit profileName; }; + serde_bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }; + sled = rustPackages."registry+https://github.com/rust-lang/crates.io-index".sled."0.34.7" { inherit profileName; }; + tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.12.0" { inherit profileName; }; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".garage_util."0.5.1" = overridableMkRustCrate (profileName: rec { name = "garage_util"; - version = "0.5.0"; + version = "0.5.1"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "1e096994382447431e2f3c70e3685eb8b24c00eceff8667bb22a2a27ff17832f"; }; + dependencies = { + blake2 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".blake2."0.9.2" { inherit profileName; }; + chrono = rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.19" { inherit profileName; }; + err_derive = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.0" { profileName = "__noProfile"; }; + futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.17" { inherit profileName; }; + hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }; + http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.5" { inherit profileName; }; + hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.13" { inherit profileName; }; + log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; }; + netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.3.0" { inherit profileName; }; + rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.4" { inherit profileName; }; + rmp_serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }; + serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.130" { inherit profileName; }; + serde_json = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.68" { inherit profileName; }; + sha2 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".sha2."0.9.8" { inherit profileName; }; + sled = rustPackages."registry+https://github.com/rust-lang/crates.io-index".sled."0.34.7" { inherit profileName; }; + tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.12.0" { inherit profileName; }; + toml = rustPackages."registry+https://github.com/rust-lang/crates.io-index".toml."0.5.8" { inherit profileName; }; + xxhash_rust = rustPackages."registry+https://github.com/rust-lang/crates.io-index".xxhash-rust."0.8.2" { inherit profileName; }; + }; + }); + + "unknown".garage_util."0.6.0" = overridableMkRustCrate (profileName: rec { + name = "garage_util"; + version = "0.6.0"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/util"); dependencies = { @@ -784,18 +890,18 @@ in }; }); - "unknown".garage_web."0.5.0" = overridableMkRustCrate (profileName: rec { + "unknown".garage_web."0.6.0" = overridableMkRustCrate (profileName: rec { name = "garage_web"; - version = "0.5.0"; + version = "0.6.0"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/web"); dependencies = { err_derive = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.0" { profileName = "__noProfile"; }; futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.17" { inherit profileName; }; - garage_api = rustPackages."unknown".garage_api."0.5.0" { inherit profileName; }; - garage_model = rustPackages."unknown".garage_model."0.5.0" { inherit profileName; }; - garage_table = rustPackages."unknown".garage_table."0.5.0" { inherit profileName; }; - garage_util = rustPackages."unknown".garage_util."0.5.0" { inherit profileName; }; + garage_api = rustPackages."unknown".garage_api."0.6.0" { inherit profileName; }; + garage_model = rustPackages."unknown".garage_model."0.6.0" { inherit profileName; }; + garage_table = rustPackages."unknown".garage_table."0.6.0" { inherit profileName; }; + garage_util = rustPackages."unknown".garage_util."0.6.0" { inherit profileName; }; http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.5" { inherit profileName; }; hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.13" { inherit profileName; }; log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; }; diff --git a/src/garage/admin.rs b/src/garage/admin.rs index 5599c53f..74b24584 100644 --- a/src/garage/admin.rs +++ b/src/garage/admin.rs @@ -19,6 +19,7 @@ use garage_model::bucket_alias_table::*; use garage_model::bucket_table::*; use garage_model::garage::Garage; use garage_model::key_table::*; +use garage_model::migrate::Migrate; use garage_model::permission::*; use crate::cli::*; @@ -31,6 +32,7 @@ pub enum AdminRpc { BucketOperation(BucketOperation), KeyOperation(KeyOperation), LaunchRepair(RepairOpt), + Migrate(MigrateOpt), Stats(StatsOpt), // Replies @@ -650,6 +652,22 @@ impl AdminRpcHandler { Ok(()) } + async fn handle_migrate(self: &Arc, opt: MigrateOpt) -> Result { + if !opt.yes { + return Err(Error::BadRpc( + "Please provide the --yes flag to initiate migration operation.".to_string(), + )); + } + + let m = Migrate { + garage: self.garage.clone(), + }; + match opt.what { + MigrateWhat::Buckets050 => m.migrate_buckets050().await, + }?; + Ok(AdminRpc::Ok("Migration successfull.".into())) + } + async fn handle_launch_repair(self: &Arc, opt: RepairOpt) -> Result { if !opt.yes { return Err(Error::BadRpc( @@ -819,6 +837,7 @@ impl EndpointHandler for AdminRpcHandler { match message { AdminRpc::BucketOperation(bo) => self.handle_bucket_cmd(bo).await, AdminRpc::KeyOperation(ko) => self.handle_key_cmd(ko).await, + AdminRpc::Migrate(opt) => self.handle_migrate(opt.clone()).await, AdminRpc::LaunchRepair(opt) => self.handle_launch_repair(opt.clone()).await, AdminRpc::Stats(opt) => self.handle_stats(opt.clone()).await, _ => Err(Error::BadRpc("Invalid RPC".to_string())), diff --git a/src/garage/cli/cmd.rs b/src/garage/cli/cmd.rs index b7508e45..b65fea02 100644 --- a/src/garage/cli/cmd.rs +++ b/src/garage/cli/cmd.rs @@ -29,6 +29,9 @@ pub async fn cli_command_dispatch( Command::Key(ko) => { cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::KeyOperation(ko)).await } + Command::Migrate(mo) => { + cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::Migrate(mo)).await + } Command::Repair(ro) => { cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::LaunchRepair(ro)).await } diff --git a/src/garage/cli/structs.rs b/src/garage/cli/structs.rs index 1905069e..bd7abc8e 100644 --- a/src/garage/cli/structs.rs +++ b/src/garage/cli/structs.rs @@ -28,6 +28,11 @@ pub enum Command { #[structopt(name = "key")] Key(KeyOperation), + /// Run migrations from previous Garage version + /// (DO NOT USE WITHOUT READING FULL DOCUMENTATION) + #[structopt(name = "migrate")] + Migrate(MigrateOpt), + /// Start repair of node data #[structopt(name = "repair")] Repair(RepairOpt), @@ -319,6 +324,23 @@ pub struct KeyImportOpt { pub name: String, } +#[derive(Serialize, Deserialize, StructOpt, Debug, Clone)] +pub struct MigrateOpt { + /// Confirm the launch of the migrate operation + #[structopt(long = "yes")] + pub yes: bool, + + #[structopt(subcommand)] + pub what: MigrateWhat, +} + +#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)] +pub enum MigrateWhat { + /// Migrate buckets and permissions from v0.5.0 + #[structopt(name = "buckets050")] + Buckets050, +} + #[derive(Serialize, Deserialize, StructOpt, Debug, Clone)] pub struct RepairOpt { /// Launch repair operation on all nodes diff --git a/src/model/Cargo.toml b/src/model/Cargo.toml index 12c08719..03881f5d 100644 --- a/src/model/Cargo.toml +++ b/src/model/Cargo.toml @@ -17,7 +17,7 @@ path = "lib.rs" garage_rpc = { version = "0.6.0", path = "../rpc" } garage_table = { version = "0.6.0", path = "../table" } garage_util = { version = "0.6.0", path = "../util" } -garage_model_050 = { package = "garage_model", version = "0.5.0" } +garage_model_050 = { package = "garage_model", version = "0.5.1" } async-trait = "0.1.7" arc-swap = "1.0" diff --git a/src/model/key_table.rs b/src/model/key_table.rs index 469dbd49..526ed496 100644 --- a/src/model/key_table.rs +++ b/src/model/key_table.rs @@ -171,4 +171,31 @@ impl TableSchema for KeyTable { } } } + + fn try_migrate(bytes: &[u8]) -> Option { + let old_k = + match rmp_serde::decode::from_read_ref::<_, garage_model_050::key_table::Key>(bytes) { + Ok(x) => x, + Err(_) => return None, + }; + let state = if old_k.deleted.get() { + crdt::Deletable::Deleted + } else { + // Authorized buckets is ignored here, + // migration is performed in specific migration code in + // garage/migrate.rs + crdt::Deletable::Present(KeyParams { + allow_create_bucket: crdt::Lww::new(false), + authorized_buckets: crdt::Map::new(), + local_aliases: crdt::LwwMap::new(), + }) + }; + let name = crdt::Lww::migrate_from_raw(old_k.name.timestamp(), old_k.name.get().clone()); + Some(Key { + key_id: old_k.key_id, + secret_key: old_k.secret_key, + name, + state, + }) + } } diff --git a/src/model/lib.rs b/src/model/lib.rs index fe8cfdad..3f6b5cd4 100644 --- a/src/model/lib.rs +++ b/src/model/lib.rs @@ -8,6 +8,7 @@ pub mod bucket_helper; pub mod bucket_table; pub mod garage; pub mod key_table; +pub mod migrate; pub mod object_table; pub mod permission; pub mod version_table; diff --git a/src/model/migrate.rs b/src/model/migrate.rs new file mode 100644 index 00000000..35ff1807 --- /dev/null +++ b/src/model/migrate.rs @@ -0,0 +1,93 @@ +use std::sync::Arc; + +use garage_table::util::EmptyKey; +use garage_util::crdt::*; +use garage_util::data::*; +use garage_util::error::*; +use garage_util::time::*; + +use garage_model_050::bucket_table as old_bucket; + +use crate::bucket_alias_table::*; +use crate::bucket_table::*; +use crate::garage::Garage; +use crate::permission::*; + +pub struct Migrate { + pub garage: Arc, +} + +impl Migrate { + pub async fn migrate_buckets050(&self) -> Result<(), Error> { + let tree = self.garage.db.open_tree("bucket:table")?; + + for res in tree.iter() { + let (_k, v) = res?; + let bucket = rmp_serde::decode::from_read_ref::<_, old_bucket::Bucket>(&v[..])?; + + if let old_bucket::BucketState::Present(p) = bucket.state.get() { + self.migrate_buckets050_do_bucket(&bucket, p).await?; + } + } + + Ok(()) + } + + pub async fn migrate_buckets050_do_bucket( + &self, + old_bucket: &old_bucket::Bucket, + old_bucket_p: &old_bucket::BucketParams, + ) -> Result<(), Error> { + let mut new_ak = Map::new(); + for (k, ts, perm) in old_bucket_p.authorized_keys.items().iter() { + new_ak.put( + k.to_string(), + BucketKeyPerm { + timestamp: *ts, + allow_read: perm.allow_read, + allow_write: perm.allow_write, + allow_owner: false, + }, + ); + } + + let mut aliases = LwwMap::new(); + aliases.update_in_place(old_bucket.name.clone(), true); + + let new_bucket = Bucket { + id: blake2sum(old_bucket.name.as_bytes()), + state: Deletable::Present(BucketParams { + creation_date: now_msec(), + authorized_keys: new_ak.clone(), + website_access: Lww::new(*old_bucket_p.website.get()), + website_config: Lww::new(None), + aliases, + local_aliases: LwwMap::new(), + }), + }; + self.garage.bucket_table.insert(&new_bucket).await?; + + let new_alias = BucketAlias { + name: old_bucket.name.clone(), + state: Lww::new(Deletable::Present(AliasParams { + bucket_id: new_bucket.id, + })), + }; + self.garage.bucket_alias_table.insert(&new_alias).await?; + + for (k, perm) in new_ak.items().iter() { + let mut key = self + .garage + .key_table + .get(&EmptyKey, k) + .await? + .ok_or_message(format!("Missing key: {}", k))?; + if let Some(p) = key.state.as_option_mut() { + p.authorized_buckets.put(new_bucket.id, *perm); + } + self.garage.key_table.insert(&key).await?; + } + + Ok(()) + } +} diff --git a/src/model/object_table.rs b/src/model/object_table.rs index 285cb5a7..45f0daf4 100644 --- a/src/model/object_table.rs +++ b/src/model/object_table.rs @@ -11,6 +11,8 @@ use garage_table::*; use crate::version_table::*; +use garage_model_050::object_table as old; + /// An object #[derive(PartialEq, Clone, Debug, Serialize, Deserialize)] pub struct Object { @@ -255,4 +257,73 @@ impl TableSchema for ObjectTable { let deleted = !entry.versions.iter().any(|v| v.is_data()); filter.apply(deleted) } + + fn try_migrate(bytes: &[u8]) -> Option { + let old_v = match rmp_serde::decode::from_read_ref::<_, old::Object>(bytes) { + Ok(x) => x, + Err(_) => return None, + }; + Some(migrate_object(old_v)) + } +} + +// vvvvvvvv migration code, stupid stuff vvvvvvvvvvvv +// (we just want to change bucket into bucket_id by hashing it) + +fn migrate_object(o: old::Object) -> Object { + let versions = o + .versions() + .iter() + .cloned() + .map(migrate_object_version) + .collect(); + Object { + bucket_id: blake2sum(o.bucket.as_bytes()), + key: o.key, + versions, + } +} + +fn migrate_object_version(v: old::ObjectVersion) -> ObjectVersion { + ObjectVersion { + uuid: Uuid::try_from(v.uuid.as_slice()).unwrap(), + timestamp: v.timestamp, + state: match v.state { + old::ObjectVersionState::Uploading(h) => { + ObjectVersionState::Uploading(migrate_object_version_headers(h)) + } + old::ObjectVersionState::Complete(d) => { + ObjectVersionState::Complete(migrate_object_version_data(d)) + } + old::ObjectVersionState::Aborted => ObjectVersionState::Aborted, + }, + } +} + +fn migrate_object_version_headers(h: old::ObjectVersionHeaders) -> ObjectVersionHeaders { + ObjectVersionHeaders { + content_type: h.content_type, + other: h.other, + } +} + +fn migrate_object_version_data(d: old::ObjectVersionData) -> ObjectVersionData { + match d { + old::ObjectVersionData::DeleteMarker => ObjectVersionData::DeleteMarker, + old::ObjectVersionData::Inline(m, b) => { + ObjectVersionData::Inline(migrate_object_version_meta(m), b) + } + old::ObjectVersionData::FirstBlock(m, h) => ObjectVersionData::FirstBlock( + migrate_object_version_meta(m), + Hash::try_from(h.as_slice()).unwrap(), + ), + } +} + +fn migrate_object_version_meta(m: old::ObjectVersionMeta) -> ObjectVersionMeta { + ObjectVersionMeta { + headers: migrate_object_version_headers(m.headers), + size: m.size, + etag: m.etag, + } } diff --git a/src/model/version_table.rs b/src/model/version_table.rs index 4edea0b7..05cae831 100644 --- a/src/model/version_table.rs +++ b/src/model/version_table.rs @@ -147,4 +147,39 @@ impl TableSchema for VersionTable { fn matches_filter(entry: &Self::E, filter: &Self::Filter) -> bool { filter.apply(entry.deleted.get()) } + + fn try_migrate(bytes: &[u8]) -> Option { + let old = + match rmp_serde::decode::from_read_ref::<_, garage_model_050::version_table::Version>( + bytes, + ) { + Ok(x) => x, + Err(_) => return None, + }; + let mut new_blocks = crdt::Map::new(); + for (k, v) in old.blocks.items().iter() { + new_blocks.put( + VersionBlockKey { + part_number: k.part_number, + offset: k.offset, + }, + VersionBlock { + hash: Hash::try_from(v.hash.as_slice()).unwrap(), + size: v.size, + }, + ); + } + let mut new_parts_etags = crdt::Map::new(); + for (k, v) in old.parts_etags.items().iter() { + new_parts_etags.put(*k, v.clone()); + } + Some(Version { + uuid: Hash::try_from(old.uuid.as_slice()).unwrap(), + deleted: crdt::Bool::new(old.deleted.get()), + blocks: new_blocks, + parts_etags: new_parts_etags, + bucket_id: blake2sum(old.bucket.as_bytes()), + key: old.key, + }) + } }