forked from Deuxfleurs/garage
make most requested changes
This commit is contained in:
parent
16300bbd89
commit
74373aebcf
20 changed files with 38 additions and 44 deletions
|
@ -29,7 +29,7 @@ pub enum Error {
|
||||||
NotFound,
|
NotFound,
|
||||||
|
|
||||||
// Category: bad request
|
// Category: bad request
|
||||||
/// The request used an invalid path
|
/// The request contained an invalid UTF-8 sequence in its path or in other parameters
|
||||||
#[error(display = "Invalid UTF-8: {}", _0)]
|
#[error(display = "Invalid UTF-8: {}", _0)]
|
||||||
InvalidUTF8Str(#[error(source)] std::str::Utf8Error),
|
InvalidUTF8Str(#[error(source)] std::str::Utf8Error),
|
||||||
|
|
||||||
|
@ -65,7 +65,7 @@ impl From<roxmltree::Error> for Error {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Error {
|
impl Error {
|
||||||
/// Convert an error into an Http status code
|
/// Get the HTTP status code that best represents the meaning of the error for the client
|
||||||
pub fn http_status_code(&self) -> StatusCode {
|
pub fn http_status_code(&self) -> StatusCode {
|
||||||
match self {
|
match self {
|
||||||
Error::NotFound => StatusCode::NOT_FOUND,
|
Error::NotFound => StatusCode::NOT_FOUND,
|
||||||
|
|
|
@ -1,4 +1,3 @@
|
||||||
#![deny(missing_crate_level_docs, missing_docs)]
|
|
||||||
//! Crate for serving a S3 compatible API
|
//! Crate for serving a S3 compatible API
|
||||||
#[macro_use]
|
#[macro_use]
|
||||||
extern crate log;
|
extern crate log;
|
||||||
|
|
|
@ -1,6 +1,5 @@
|
||||||
#![deny(missing_crate_level_docs, missing_docs)]
|
|
||||||
#![recursion_limit = "1024"]
|
#![recursion_limit = "1024"]
|
||||||
//! Garage CLI, used to interact with a running Garage instance, and to launch a Garage
|
//! Garage CLI, used to interact with a running Garage instance, and to launch a Garage instance
|
||||||
|
|
||||||
#[macro_use]
|
#[macro_use]
|
||||||
extern crate log;
|
extern crate log;
|
||||||
|
|
|
@ -157,7 +157,7 @@ impl BlockManager {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Write a block to disk
|
/// Write a block to disk
|
||||||
pub async fn write_block(&self, hash: &Hash, data: &[u8]) -> Result<Message, Error> {
|
async fn write_block(&self, hash: &Hash, data: &[u8]) -> Result<Message, Error> {
|
||||||
let _lock = self.data_dir_lock.lock().await;
|
let _lock = self.data_dir_lock.lock().await;
|
||||||
|
|
||||||
let mut path = self.block_dir(hash);
|
let mut path = self.block_dir(hash);
|
||||||
|
@ -176,7 +176,7 @@ impl BlockManager {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Read block from disk, verifying it's integrity
|
/// Read block from disk, verifying it's integrity
|
||||||
pub async fn read_block(&self, hash: &Hash) -> Result<Message, Error> {
|
async fn read_block(&self, hash: &Hash) -> Result<Message, Error> {
|
||||||
let path = self.block_path(hash);
|
let path = self.block_path(hash);
|
||||||
|
|
||||||
let mut f = match fs::File::open(&path).await {
|
let mut f = match fs::File::open(&path).await {
|
||||||
|
@ -208,7 +208,7 @@ impl BlockManager {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Check if this node should have a block, but don't actually have it
|
/// Check if this node should have a block, but don't actually have it
|
||||||
pub async fn need_block(&self, hash: &Hash) -> Result<bool, Error> {
|
async fn need_block(&self, hash: &Hash) -> Result<bool, Error> {
|
||||||
let needed = self
|
let needed = self
|
||||||
.rc
|
.rc
|
||||||
.get(hash.as_ref())?
|
.get(hash.as_ref())?
|
||||||
|
|
|
@ -10,16 +10,14 @@ use crate::block::*;
|
||||||
|
|
||||||
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
||||||
pub struct BlockRef {
|
pub struct BlockRef {
|
||||||
// Primary key
|
/// Hash of the block, used as partition key
|
||||||
/// Hash of the block
|
|
||||||
pub block: Hash,
|
pub block: Hash,
|
||||||
|
|
||||||
// Sort key
|
/// Id of the Version for the object containing this block, used as sorting key
|
||||||
// why a version on a hashed (probably immutable) piece of data?
|
|
||||||
pub version: UUID,
|
pub version: UUID,
|
||||||
|
|
||||||
// Keep track of deleted status
|
// Keep track of deleted status
|
||||||
/// Is that block deleted
|
/// Is the Version that contains this block deleted
|
||||||
pub deleted: crdt::Bool,
|
pub deleted: crdt::Bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -6,7 +6,7 @@ use garage_table::*;
|
||||||
/// An api key
|
/// An api key
|
||||||
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
||||||
pub struct Key {
|
pub struct Key {
|
||||||
/// The id of the key (immutable)
|
/// The id of the key (immutable), used as partition key
|
||||||
pub key_id: String,
|
pub key_id: String,
|
||||||
|
|
||||||
/// The secret_key associated
|
/// The secret_key associated
|
||||||
|
@ -19,6 +19,7 @@ pub struct Key {
|
||||||
pub deleted: crdt::Bool,
|
pub deleted: crdt::Bool,
|
||||||
|
|
||||||
/// Buckets in which the key is authorized. Empty if `Key` is deleted
|
/// Buckets in which the key is authorized. Empty if `Key` is deleted
|
||||||
|
// CRDT interaction: deleted implies authorized_buckets is empty
|
||||||
pub authorized_buckets: crdt::LWWMap<String, PermissionSet>,
|
pub authorized_buckets: crdt::LWWMap<String, PermissionSet>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,4 +1,3 @@
|
||||||
#![warn(missing_docs)]
|
|
||||||
#[macro_use]
|
#[macro_use]
|
||||||
extern crate log;
|
extern crate log;
|
||||||
|
|
||||||
|
|
|
@ -14,13 +14,13 @@ use crate::version_table::*;
|
||||||
/// An object
|
/// An object
|
||||||
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
||||||
pub struct Object {
|
pub struct Object {
|
||||||
/// The bucket in which the object is stored
|
/// The bucket in which the object is stored, used as partition key
|
||||||
pub bucket: String,
|
pub bucket: String,
|
||||||
|
|
||||||
/// The key at which the object is stored in its bucket
|
/// The key at which the object is stored in its bucket, used as sorting key
|
||||||
pub key: String,
|
pub key: String,
|
||||||
|
|
||||||
/// The list of known versions of the object
|
/// The list of currenty stored versions of the object
|
||||||
versions: Vec<ObjectVersion>,
|
versions: Vec<ObjectVersion>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -53,7 +53,7 @@ impl Object {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get a list of all versions known for `Object`
|
/// Get a list of currently stored versions of `Object`
|
||||||
pub fn versions(&self) -> &[ObjectVersion] {
|
pub fn versions(&self) -> &[ObjectVersion] {
|
||||||
&self.versions[..]
|
&self.versions[..]
|
||||||
}
|
}
|
||||||
|
@ -77,7 +77,7 @@ pub enum ObjectVersionState {
|
||||||
Uploading(ObjectVersionHeaders),
|
Uploading(ObjectVersionHeaders),
|
||||||
/// The version is fully received
|
/// The version is fully received
|
||||||
Complete(ObjectVersionData),
|
Complete(ObjectVersionData),
|
||||||
/// The version was never fully received
|
/// The version uploaded containded errors or the upload was explicitly aborted
|
||||||
Aborted,
|
Aborted,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -105,7 +105,7 @@ impl CRDT for ObjectVersionState {
|
||||||
/// Data about an object version
|
/// Data about an object version
|
||||||
#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
|
#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
|
||||||
pub enum ObjectVersionData {
|
pub enum ObjectVersionData {
|
||||||
/// The version is deleted
|
/// The object was deleted, this Version is a tombstone to mark it as such
|
||||||
DeleteMarker,
|
DeleteMarker,
|
||||||
/// The object is short, it's stored inlined
|
/// The object is short, it's stored inlined
|
||||||
Inline(ObjectVersionMeta, #[serde(with = "serde_bytes")] Vec<u8>),
|
Inline(ObjectVersionMeta, #[serde(with = "serde_bytes")] Vec<u8>),
|
||||||
|
@ -159,7 +159,7 @@ impl ObjectVersion {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Is the object version available (received and not deleted)
|
/// Is the object version available (received and not a tombstone)
|
||||||
pub fn is_data(&self) -> bool {
|
pub fn is_data(&self) -> bool {
|
||||||
match self.state {
|
match self.state {
|
||||||
ObjectVersionState::Complete(ObjectVersionData::DeleteMarker) => false,
|
ObjectVersionState::Complete(ObjectVersionData::DeleteMarker) => false,
|
||||||
|
|
|
@ -13,7 +13,7 @@ use crate::block_ref_table::*;
|
||||||
/// A version of an object
|
/// A version of an object
|
||||||
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
||||||
pub struct Version {
|
pub struct Version {
|
||||||
/// UUID of the version
|
/// UUID of the version, used as partition key
|
||||||
pub uuid: UUID,
|
pub uuid: UUID,
|
||||||
|
|
||||||
// Actual data: the blocks for this version
|
// Actual data: the blocks for this version
|
||||||
|
@ -49,9 +49,9 @@ impl Version {
|
||||||
|
|
||||||
#[derive(PartialEq, Eq, Clone, Copy, Debug, Serialize, Deserialize)]
|
#[derive(PartialEq, Eq, Clone, Copy, Debug, Serialize, Deserialize)]
|
||||||
pub struct VersionBlockKey {
|
pub struct VersionBlockKey {
|
||||||
/// Number of the part, starting at 1
|
/// Number of the part
|
||||||
pub part_number: u64,
|
pub part_number: u64,
|
||||||
/// offset of the block in the file, starting at 0
|
/// Offset of this sub-segment in its part
|
||||||
pub offset: u64,
|
pub offset: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,4 +1,3 @@
|
||||||
#![warn(missing_crate_level_docs, missing_docs)]
|
|
||||||
//! Crate containing rpc related functions and types used in Garage
|
//! Crate containing rpc related functions and types used in Garage
|
||||||
|
|
||||||
#[macro_use]
|
#[macro_use]
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
/// Module containing structs related to membership management
|
//! Module containing structs related to membership management
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::fmt::Write as FmtWrite;
|
use std::fmt::Write as FmtWrite;
|
||||||
use std::io::{Read, Write};
|
use std::io::{Read, Write};
|
||||||
|
@ -96,7 +96,7 @@ pub struct System {
|
||||||
rpc_client: Arc<RpcClient<Message>>,
|
rpc_client: Arc<RpcClient<Message>>,
|
||||||
|
|
||||||
pub(crate) status: watch::Receiver<Arc<Status>>,
|
pub(crate) status: watch::Receiver<Arc<Status>>,
|
||||||
/// The ring, viewed by this node
|
/// The ring
|
||||||
pub ring: watch::Receiver<Arc<Ring>>,
|
pub ring: watch::Receiver<Arc<Ring>>,
|
||||||
|
|
||||||
update_lock: Mutex<Updaters>,
|
update_lock: Mutex<Updaters>,
|
||||||
|
@ -114,9 +114,8 @@ struct Updaters {
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct Status {
|
pub struct Status {
|
||||||
/// Mapping of each node id to its known status
|
/// Mapping of each node id to its known status
|
||||||
// considering its sorted regularly, maybe it should be a btreeset?
|
|
||||||
pub nodes: HashMap<UUID, Arc<StatusEntry>>,
|
pub nodes: HashMap<UUID, Arc<StatusEntry>>,
|
||||||
/// Hash of this entry
|
/// Hash of `nodes`, used to detect when nodes have different views of the cluster
|
||||||
pub hash: Hash,
|
pub hash: Hash,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -380,7 +379,7 @@ impl System {
|
||||||
id_option,
|
id_option,
|
||||||
addr,
|
addr,
|
||||||
sys.rpc_client
|
sys.rpc_client
|
||||||
.get_addr()
|
.by_addr()
|
||||||
.call(&addr, ping_msg_ref, PING_TIMEOUT)
|
.call(&addr, ping_msg_ref, PING_TIMEOUT)
|
||||||
.await,
|
.await,
|
||||||
)
|
)
|
||||||
|
@ -418,6 +417,8 @@ impl System {
|
||||||
}
|
}
|
||||||
} else if let Some(id) = id_option {
|
} else if let Some(id) = id_option {
|
||||||
if let Some(st) = status.nodes.get_mut(id) {
|
if let Some(st) = status.nodes.get_mut(id) {
|
||||||
|
// TODO this might double-increment the value as the counter is already
|
||||||
|
// incremented for any kind of failure in rpc_client
|
||||||
st.num_failures.fetch_add(1, Ordering::SeqCst);
|
st.num_failures.fetch_add(1, Ordering::SeqCst);
|
||||||
if !st.is_up() {
|
if !st.is_up() {
|
||||||
warn!("Node {:?} seems to be down.", id);
|
warn!("Node {:?} seems to be down.", id);
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
//! Module containing types related to computing nodes which should receive a copy of data blocks
|
//! Module containing types related to computing nodes which should receive a copy of data blocks
|
||||||
|
//! and metadata
|
||||||
use std::collections::{HashMap, HashSet};
|
use std::collections::{HashMap, HashSet};
|
||||||
use std::convert::TryInto;
|
use std::convert::TryInto;
|
||||||
|
|
||||||
|
@ -197,7 +198,7 @@ impl Ring {
|
||||||
top >> (16 - PARTITION_BITS)
|
top >> (16 - PARTITION_BITS)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get the list of partitions and
|
/// Get the list of partitions and the first hash of a partition key that would fall in it
|
||||||
pub fn partitions(&self) -> Vec<(Partition, Hash)> {
|
pub fn partitions(&self) -> Vec<(Partition, Hash)> {
|
||||||
let mut ret = vec![];
|
let mut ret = vec![];
|
||||||
|
|
||||||
|
@ -211,6 +212,7 @@ impl Ring {
|
||||||
ret
|
ret
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TODO rename this function as it no longer walk the ring
|
||||||
/// Walk the ring to find the n servers in which data should be replicated
|
/// Walk the ring to find the n servers in which data should be replicated
|
||||||
pub fn walk_ring(&self, from: &Hash, n: usize) -> Vec<UUID> {
|
pub fn walk_ring(&self, from: &Hash, n: usize) -> Vec<UUID> {
|
||||||
if self.ring.len() != 1 << PARTITION_BITS {
|
if self.ring.len() != 1 << PARTITION_BITS {
|
||||||
|
|
|
@ -53,6 +53,7 @@ impl RequestStrategy {
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
/// Set if requests can be dropped after quorum has been reached
|
/// Set if requests can be dropped after quorum has been reached
|
||||||
|
/// In general true for read requests, and false for write
|
||||||
pub fn interrupt_after_quorum(mut self, interrupt: bool) -> Self {
|
pub fn interrupt_after_quorum(mut self, interrupt: bool) -> Self {
|
||||||
self.rs_interrupt_after_quorum = interrupt;
|
self.rs_interrupt_after_quorum = interrupt;
|
||||||
self
|
self
|
||||||
|
@ -103,8 +104,8 @@ impl<M: RpcMessage + 'static> RpcClient<M> {
|
||||||
self.local_handler.swap(Some(Arc::new((my_id, handler))));
|
self.local_handler.swap(Some(Arc::new((my_id, handler))));
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get the server address this client connect to
|
/// Get a RPC client to make calls using node's SocketAddr instead of its ID
|
||||||
pub fn get_addr(&self) -> &RpcAddrClient<M> {
|
pub fn by_addr(&self) -> &RpcAddrClient<M> {
|
||||||
&self.rpc_addr_client
|
&self.rpc_addr_client
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -166,7 +167,7 @@ impl<M: RpcMessage + 'static> RpcClient<M> {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Make a RPC call to multiple servers, returning either a Vec of responses, or an error if
|
/// Make a RPC call to multiple servers, returning either a Vec of responses, or an error if
|
||||||
/// strategy could not be respected due to to many errors
|
/// strategy could not be respected due to too many errors
|
||||||
pub async fn try_call_many(
|
pub async fn try_call_many(
|
||||||
self: &Arc<Self>,
|
self: &Arc<Self>,
|
||||||
to: &[UUID],
|
to: &[UUID],
|
||||||
|
@ -226,7 +227,7 @@ impl<M: RpcMessage + 'static> RpcClient<M> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Endpoint to which send RPC
|
/// Thin wrapper arround an `RpcHttpClient` specifying the path of the request
|
||||||
pub struct RpcAddrClient<M: RpcMessage> {
|
pub struct RpcAddrClient<M: RpcMessage> {
|
||||||
phantom: PhantomData<M>,
|
phantom: PhantomData<M>,
|
||||||
|
|
||||||
|
|
|
@ -1,4 +1,3 @@
|
||||||
#![warn(missing_docs)]
|
|
||||||
#![recursion_limit = "1024"]
|
#![recursion_limit = "1024"]
|
||||||
|
|
||||||
#[macro_use]
|
#[macro_use]
|
||||||
|
|
|
@ -16,7 +16,6 @@ pub trait TableReplication: Send + Sync {
|
||||||
fn write_nodes(&self, hash: &Hash) -> Vec<UUID>;
|
fn write_nodes(&self, hash: &Hash) -> Vec<UUID>;
|
||||||
/// Responses needed to consider a write succesfull
|
/// Responses needed to consider a write succesfull
|
||||||
fn write_quorum(&self) -> usize;
|
fn write_quorum(&self) -> usize;
|
||||||
// this feels like its write_nodes().len() - write_quorum()
|
|
||||||
fn max_write_errors(&self) -> usize;
|
fn max_write_errors(&self) -> usize;
|
||||||
|
|
||||||
// Accessing partitions, for Merkle tree & sync
|
// Accessing partitions, for Merkle tree & sync
|
||||||
|
|
|
@ -4,7 +4,7 @@ use garage_util::data::*;
|
||||||
|
|
||||||
use crate::crdt::CRDT;
|
use crate::crdt::CRDT;
|
||||||
|
|
||||||
/// Trait for partitionnable data
|
/// Trait for field used to partition data
|
||||||
pub trait PartitionKey {
|
pub trait PartitionKey {
|
||||||
/// Get the key used to partition
|
/// Get the key used to partition
|
||||||
fn hash(&self) -> Hash;
|
fn hash(&self) -> Hash;
|
||||||
|
@ -22,7 +22,7 @@ impl PartitionKey for Hash {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Trait for sortable data
|
/// Trait for field used to sort data
|
||||||
pub trait SortKey {
|
pub trait SortKey {
|
||||||
/// Get the key used to sort
|
/// Get the key used to sort
|
||||||
fn sort_key(&self) -> &[u8];
|
fn sort_key(&self) -> &[u8];
|
||||||
|
|
|
@ -1,5 +1,4 @@
|
||||||
//! Module containing error types used in Garage
|
//! Module containing error types used in Garage
|
||||||
#![allow(missing_docs)]
|
|
||||||
use err_derive::Error;
|
use err_derive::Error;
|
||||||
use hyper::StatusCode;
|
use hyper::StatusCode;
|
||||||
use std::io;
|
use std::io;
|
||||||
|
|
|
@ -1,4 +1,3 @@
|
||||||
#![warn(missing_crate_level_docs, missing_docs)]
|
|
||||||
//! Crate containing common functions and types used in Garage
|
//! Crate containing common functions and types used in Garage
|
||||||
|
|
||||||
#[macro_use]
|
#[macro_use]
|
||||||
|
|
|
@ -19,7 +19,7 @@ pub enum Error {
|
||||||
#[error(display = "Not found")]
|
#[error(display = "Not found")]
|
||||||
NotFound,
|
NotFound,
|
||||||
|
|
||||||
/// The client requested a malformed path
|
/// The request contained an invalid UTF-8 sequence in its path or in other parameters
|
||||||
#[error(display = "Invalid UTF-8: {}", _0)]
|
#[error(display = "Invalid UTF-8: {}", _0)]
|
||||||
InvalidUTF8(#[error(source)] std::str::Utf8Error),
|
InvalidUTF8(#[error(source)] std::str::Utf8Error),
|
||||||
|
|
||||||
|
|
|
@ -1,4 +1,3 @@
|
||||||
#![deny(missing_crate_level_docs, missing_docs)]
|
|
||||||
//! Crate for handling web serving of s3 bucket
|
//! Crate for handling web serving of s3 bucket
|
||||||
#[macro_use]
|
#[macro_use]
|
||||||
extern crate log;
|
extern crate log;
|
||||||
|
|
Loading…
Reference in a new issue