From b9127dd6f88d3c66181ef6748167efd94283ff41 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 28 May 2021 13:58:47 +0200 Subject: [PATCH] Prepare for v0.3.0 and add migration path from v0.2.1.x --- Cargo.lock | 83 +++++++++++++++++++++++++++++++++++-------- Makefile | 17 +-------- config.dev.toml | 22 ------------ make_docker.sh | 20 +++++++++++ src/api/Cargo.toml | 8 ++--- src/garage/Cargo.toml | 14 ++++---- src/garage/cli.rs | 8 ++--- src/model/Cargo.toml | 8 ++--- src/rpc/Cargo.toml | 6 ++-- src/rpc/membership.rs | 20 ++++++++--- src/rpc/ring.rs | 28 +++++++++++++-- src/table/Cargo.toml | 6 ++-- src/util/Cargo.toml | 2 +- src/web/Cargo.toml | 10 +++--- 14 files changed, 161 insertions(+), 91 deletions(-) delete mode 100644 config.dev.toml create mode 100755 make_docker.sh diff --git a/Cargo.lock b/Cargo.lock index 1ea13711..c2bec83e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -350,16 +350,16 @@ dependencies = [ [[package]] name = "garage" -version = "0.2.1" +version = "0.3.0" dependencies = [ "bytes", "futures", "futures-util", "garage_api", "garage_model", - "garage_rpc", + "garage_rpc 0.3.0", "garage_table", - "garage_util", + "garage_util 0.3.0", "garage_web", "git-version", "hex", @@ -376,7 +376,7 @@ dependencies = [ [[package]] name = "garage_api" -version = "0.2.1" +version = "0.3.0" dependencies = [ "base64", "bytes", @@ -387,7 +387,7 @@ dependencies = [ "futures-util", "garage_model", "garage_table", - "garage_util", + "garage_util 0.3.0", "hex", "hmac", "http", @@ -407,14 +407,14 @@ dependencies = [ [[package]] name = "garage_model" -version = "0.2.1" +version = "0.3.0" dependencies = [ "arc-swap", "futures", "futures-util", - "garage_rpc", + "garage_rpc 0.3.0", "garage_table", - "garage_util", + "garage_util 0.3.0", "hex", "log", "rand", @@ -428,12 +428,40 @@ dependencies = [ [[package]] name = "garage_rpc" version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c182633cebe4abed9594afb14770fc45402513765d38a4b19659ae0ccb2a2f" dependencies = [ "arc-swap", "bytes", "futures", "futures-util", - "garage_util", + "garage_util 0.2.1", + "gethostname", + "hex", + "http", + "hyper", + "hyper-rustls", + "log", + "rmp-serde", + "rustls", + "serde", + "serde_json", + "tokio", + "tokio-rustls", + "tokio-stream", + "webpki", +] + +[[package]] +name = "garage_rpc" +version = "0.3.0" +dependencies = [ + "arc-swap", + "bytes", + "futures", + "futures-util", + "garage_rpc 0.2.1", + "garage_util 0.3.0", "gethostname", "hex", "http", @@ -452,13 +480,13 @@ dependencies = [ [[package]] name = "garage_table" -version = "0.2.1" +version = "0.3.0" dependencies = [ "bytes", "futures", "futures-util", - "garage_rpc", - "garage_util", + "garage_rpc 0.3.0", + "garage_util 0.3.0", "hexdump", "log", "rand", @@ -472,6 +500,33 @@ dependencies = [ [[package]] name = "garage_util" version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef76d3779e406a16fdcaffe8d86b8ae2943a549d2b33f2c20930838764464c0" +dependencies = [ + "blake2", + "chrono", + "err-derive", + "futures", + "hex", + "http", + "hyper", + "log", + "rand", + "rmp-serde", + "rustls", + "serde", + "serde_json", + "sha2", + "sled", + "tokio", + "toml", + "webpki", + "xxhash-rust", +] + +[[package]] +name = "garage_util" +version = "0.3.0" dependencies = [ "blake2", "chrono", @@ -496,14 +551,14 @@ dependencies = [ [[package]] name = "garage_web" -version = "0.2.1" +version = "0.3.0" dependencies = [ "err-derive", "futures", "garage_api", "garage_model", "garage_table", - "garage_util", + "garage_util 0.3.0", "http", "hyper", "idna", diff --git a/Makefile b/Makefile index 51b77699..1098d712 100644 --- a/Makefile +++ b/Makefile @@ -1,20 +1,5 @@ -BIN=target/release/garage -DOCKER=lxpz/garage_amd64 - all: clear; cargo build -$(BIN): +release: RUSTFLAGS="-C link-arg=-fuse-ld=lld -C target-cpu=x86-64 -C target-feature=+sse2" cargo build --release --no-default-features - -$(BIN).stripped: $(BIN) - cp $^ $@ - strip $@ - -docker: $(BIN).stripped - docker pull archlinux:latest - docker build -t $(DOCKER):$(TAG) . - docker push $(DOCKER):$(TAG) - docker tag $(DOCKER):$(TAG) $(DOCKER):latest - docker push $(DOCKER):latest - diff --git a/config.dev.toml b/config.dev.toml deleted file mode 100644 index 215bc50c..00000000 --- a/config.dev.toml +++ /dev/null @@ -1,22 +0,0 @@ -block_size = 1048576 # objects are split in blocks of maximum this number of bytes - -metadata_dir = "/tmp/garage-meta" -data_dir = "/tmp/garage-data" - -rpc_bind_addr = "[::]:3901" # the port other Garage nodes will use to talk to this node - -bootstrap_peers = [] - -max_concurrent_rpc_requests = 12 -data_replication_factor = 3 -meta_replication_factor = 3 -meta_epidemic_fanout = 3 - -[s3_api] -api_bind_addr = "[::1]:3900" # the S3 API port, HTTP without TLS. Add a reverse proxy for the TLS part. -s3_region = "garage" # set this to anything. S3 API calls will fail if they are not made against the region set here. - -[s3_web] -bind_addr = "[::1]:3902" -root_domain = ".garage.tld" -index = "index.html" diff --git a/make_docker.sh b/make_docker.sh new file mode 100755 index 00000000..c338d0a6 --- /dev/null +++ b/make_docker.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +BIN=target/release/garage +DOCKER=lxpz/garage_amd64 + +TAG=$1 +if [ -z "$1" ]; then + echo "Usage: $0 " + exit 1 +fi + +RUSTFLAGS="-C link-arg=-fuse-ld=lld -C target-cpu=x86-64 -C target-feature=+sse2" cargo build --release --no-default-features +cp $BIN $BIN.stripped +strip $BIN.stripped + +docker pull archlinux:latest +docker build -t $DOCKER:$TAG . +docker push $DOCKER:$TAG +docker tag $DOCKER:$TAG $DOCKER:latest +docker push $DOCKER:latest diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index b9fc4bfc..8b5118cd 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_api" -version = "0.2.1" +version = "0.3.0" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" @@ -13,9 +13,9 @@ path = "lib.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -garage_model = { version = "0.2.1", path = "../model" } -garage_table = { version = "0.2.1", path = "../table" } -garage_util = { version = "0.2.1", path = "../util" } +garage_model = { version = "0.3.0", path = "../model" } +garage_table = { version = "0.3.0", path = "../table" } +garage_util = { version = "0.3.0", path = "../util" } base64 = "0.13" bytes = "1.0" diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index b6fc139b..09ed3e1e 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage" -version = "0.2.1" +version = "0.3.0" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" @@ -14,12 +14,12 @@ path = "main.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -garage_api = { version = "0.2.1", path = "../api" } -garage_model = { version = "0.2.1", path = "../model" } -garage_rpc = { version = "0.2.1", path = "../rpc" } -garage_table = { version = "0.2.1", path = "../table" } -garage_util = { version = "0.2.1", path = "../util" } -garage_web = { version = "0.2.1", path = "../web" } +garage_api = { version = "0.3.0", path = "../api" } +garage_model = { version = "0.3.0", path = "../model" } +garage_rpc = { version = "0.3.0", path = "../rpc" } +garage_table = { version = "0.3.0", path = "../table" } +garage_util = { version = "0.3.0", path = "../util" } +garage_web = { version = "0.3.0", path = "../web" } bytes = "1.0" git-version = "0.3.4" diff --git a/src/garage/cli.rs b/src/garage/cli.rs index 30d5726e..0ddb5a1c 100644 --- a/src/garage/cli.rs +++ b/src/garage/cli.rs @@ -467,9 +467,7 @@ pub async fn cmd_configure( "Please specify a capacity with the -c flag, or set node explicitly as gateway with -g".into())), }; NetworkConfigEntry { - zone: args - .zone - .expect("Please specifiy a zone with the -z flag"), + zone: args.zone.expect("Please specifiy a zone with the -z flag"), capacity, tag: args.tag.unwrap_or_default(), } @@ -481,9 +479,7 @@ pub async fn cmd_configure( _ => old.capacity, }; NetworkConfigEntry { - zone: args - .zone - .unwrap_or_else(|| old.zone.to_string()), + zone: args.zone.unwrap_or_else(|| old.zone.to_string()), capacity, tag: args.tag.unwrap_or_else(|| old.tag.to_string()), } diff --git a/src/model/Cargo.toml b/src/model/Cargo.toml index 77084531..4d5d7f9d 100644 --- a/src/model/Cargo.toml +++ b/src/model/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_model" -version = "0.2.1" +version = "0.3.0" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" @@ -13,9 +13,9 @@ path = "lib.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -garage_rpc = { version = "0.2.1", path = "../rpc" } -garage_table = { version = "0.2.1", path = "../table" } -garage_util = { version = "0.2.1", path = "../util" } +garage_rpc = { version = "0.3.0", path = "../rpc" } +garage_table = { version = "0.3.0", path = "../table" } +garage_util = { version = "0.3.0", path = "../util" } arc-swap = "1.0" hex = "0.4" diff --git a/src/rpc/Cargo.toml b/src/rpc/Cargo.toml index 622f8ebc..f1204cdf 100644 --- a/src/rpc/Cargo.toml +++ b/src/rpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_rpc" -version = "0.2.1" +version = "0.3.0" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" @@ -13,7 +13,9 @@ path = "lib.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -garage_util = { version = "0.2.1", path = "../util" } +garage_util = { version = "0.3.0", path = "../util" } + +garage_rpc_021 = { package = "garage_rpc", version = "0.2.1" } arc-swap = "1.0" bytes = "1.0" diff --git a/src/rpc/membership.rs b/src/rpc/membership.rs index 37cf8105..72477539 100644 --- a/src/rpc/membership.rs +++ b/src/rpc/membership.rs @@ -240,11 +240,21 @@ impl System { let net_config = match persist_config.load() { Ok(x) => x, Err(e) => { - info!( - "No valid previous network configuration stored ({}), starting fresh.", - e - ); - NetworkConfig::new() + match Persister::::new( + &metadata_dir, + "network_config", + ) + .load() + { + Ok(old_config) => NetworkConfig::migrate_from_021(old_config), + Err(e2) => { + info!( + "No valid previous network configuration stored ({}, {}), starting fresh.", + e, e2 + ); + NetworkConfig::new() + } + } } }; diff --git a/src/rpc/ring.rs b/src/rpc/ring.rs index a5e5d4af..90db8fd2 100644 --- a/src/rpc/ring.rs +++ b/src/rpc/ring.rs @@ -38,6 +38,31 @@ impl NetworkConfig { version: 0, } } + + pub(crate) fn migrate_from_021(old: garage_rpc_021::ring::NetworkConfig) -> Self { + let members = old + .members + .into_iter() + .map(|(id, conf)| { + ( + Hash::try_from(id.as_slice()).unwrap(), + NetworkConfigEntry { + zone: conf.datacenter, + capacity: if conf.capacity == 0 { + None + } else { + Some(conf.capacity) + }, + tag: conf.tag, + }, + ) + }) + .collect(); + Self { + members, + version: old.version, + } + } } /// The overall configuration of one (possibly remote) node @@ -178,8 +203,7 @@ impl Ring { .iter() .map(|(_id, info)| info.zone.as_str()) .collect::>(); - if (p_zns.len() < n_zones - && !p_zns.contains(&node_info.zone.as_str())) + if (p_zns.len() < n_zones && !p_zns.contains(&node_info.zone.as_str())) || (p_zns.len() == n_zones && !partitions[qv].iter().any(|(id, _i)| id == node_id)) { diff --git a/src/table/Cargo.toml b/src/table/Cargo.toml index 9b7e905c..ccbd1748 100644 --- a/src/table/Cargo.toml +++ b/src/table/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_table" -version = "0.2.1" +version = "0.3.0" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" @@ -13,8 +13,8 @@ path = "lib.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -garage_rpc = { version = "0.2.1", path = "../rpc" } -garage_util = { version = "0.2.1", path = "../util" } +garage_rpc = { version = "0.3.0", path = "../rpc" } +garage_util = { version = "0.3.0", path = "../util" } bytes = "1.0" hexdump = "0.1" diff --git a/src/util/Cargo.toml b/src/util/Cargo.toml index 79611752..91e0b2b9 100644 --- a/src/util/Cargo.toml +++ b/src/util/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_util" -version = "0.2.1" +version = "0.3.0" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" diff --git a/src/web/Cargo.toml b/src/web/Cargo.toml index 6c3b3a9a..dc31c1b4 100644 --- a/src/web/Cargo.toml +++ b/src/web/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_web" -version = "0.2.1" +version = "0.3.0" authors = ["Alex Auvolat ", "Quentin Dufour "] edition = "2018" license = "AGPL-3.0" @@ -13,10 +13,10 @@ path = "lib.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -garage_api = { version = "0.2.1", path = "../api" } -garage_model = { version = "0.2.1", path = "../model" } -garage_util = { version = "0.2.1", path = "../util" } -garage_table = { version = "0.2.1", path = "../table" } +garage_api = { version = "0.3.0", path = "../api" } +garage_model = { version = "0.3.0", path = "../model" } +garage_util = { version = "0.3.0", path = "../util" } +garage_table = { version = "0.3.0", path = "../table" } err-derive = "0.3" idna = "0.2"