Compression

This commit is contained in:
Alex 2021-12-09 15:43:19 +01:00
parent e4942490ee
commit 9b30f2b7d1
No known key found for this signature in database
GPG key ID: EDABF9711E244EB1
5 changed files with 383 additions and 68 deletions

224
Cargo.lock generated
View file

@ -2,6 +2,16 @@
# It is not intended for manual editing.
version = 3
[[package]]
name = "accept-encoding-fork"
version = "0.2.0-alpha.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "104af0beedb34a7590b5cd62c3965e89a405dfc4ac88f9704ebbeaf8b0db4597"
dependencies = [
"failure",
"http 0.2.5",
]
[[package]]
name = "acme-micro"
version = "0.12.0"
@ -19,6 +29,21 @@ dependencies = [
"ureq",
]
[[package]]
name = "addr2line"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b"
dependencies = [
"gimli",
]
[[package]]
name = "adler"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
[[package]]
name = "aho-corasick"
version = "0.7.18"
@ -28,6 +53,21 @@ dependencies = [
"memchr",
]
[[package]]
name = "alloc-no-stdlib"
version = "2.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35ef4730490ad1c4eae5c4325b2a95f521d023e5c885853ff7aca0a6a1631db3"
[[package]]
name = "alloc-stdlib"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "697ed7edc0f1711de49ce108c541623a0af97c6c60b2f6e2b65229847ac843c2"
dependencies = [
"alloc-no-stdlib",
]
[[package]]
name = "ansi_term"
version = "0.12.1"
@ -43,6 +83,22 @@ version = "1.0.51"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b26702f315f53b6071259e15dd9d64528213b44d61de1ec926eca7715d62203"
[[package]]
name = "async-compression"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5443ccbb270374a2b1055fc72da40e1f237809cd6bb0e97e66d264cd138473a6"
dependencies = [
"brotli",
"flate2",
"futures-core",
"memchr",
"pin-project-lite",
"tokio 1.14.0",
"zstd",
"zstd-safe",
]
[[package]]
name = "atty"
version = "0.2.14"
@ -60,6 +116,21 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
[[package]]
name = "backtrace"
version = "0.3.63"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "321629d8ba6513061f26707241fa9bc89524ff1cd7a915a97ef0c62c666ce1b6"
dependencies = [
"addr2line",
"cc",
"cfg-if 1.0.0",
"libc",
"miniz_oxide",
"object",
"rustc-demangle",
]
[[package]]
name = "base-x"
version = "0.2.8"
@ -78,6 +149,27 @@ version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "brotli"
version = "3.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71cb90ade945043d3d53597b2fc359bb063db8ade2bcffe7997351d0756e9d50"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
"brotli-decompressor",
]
[[package]]
name = "brotli-decompressor"
version = "2.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59ad2d4653bf5ca36ae797b1f4bb4dbddb60ce49ca4aed8a2ce4829f60425b80"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
]
[[package]]
name = "bumpalo"
version = "3.8.0"
@ -112,6 +204,9 @@ name = "cc"
version = "1.0.72"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22a9137b95ea06864e018375b72adfb7db6e6f68cfc8df5a04d00288050485ee"
dependencies = [
"jobserver",
]
[[package]]
name = "cfg-if"
@ -218,6 +313,15 @@ version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
[[package]]
name = "crc32fast"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "738c290dfaea84fc1ca15ad9c168d083b05a714e1efddd8edaab678dc28d2836"
dependencies = [
"cfg-if 1.0.0",
]
[[package]]
name = "crossbeam-deque"
version = "0.7.4"
@ -309,6 +413,40 @@ dependencies = [
"serde",
]
[[package]]
name = "failure"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86"
dependencies = [
"backtrace",
"failure_derive",
]
[[package]]
name = "failure_derive"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4"
dependencies = [
"proc-macro2",
"quote",
"syn",
"synstructure",
]
[[package]]
name = "flate2"
version = "1.0.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f"
dependencies = [
"cfg-if 1.0.0",
"crc32fast",
"libc",
"miniz_oxide",
]
[[package]]
name = "fnv"
version = "1.0.7"
@ -472,6 +610,12 @@ dependencies = [
"wasi",
]
[[package]]
name = "gimli"
version = "0.26.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78cc372d058dcf6d5ecd98510e7fbc9e5aec4d21de70f65fea8fecebcd881bd4"
[[package]]
name = "glob"
version = "0.3.0"
@ -741,6 +885,15 @@ version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
[[package]]
name = "jobserver"
version = "0.1.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa"
dependencies = [
"libc",
]
[[package]]
name = "js-sys"
version = "0.3.55"
@ -823,6 +976,16 @@ version = "0.3.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
[[package]]
name = "miniz_oxide"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b"
dependencies = [
"adler",
"autocfg",
]
[[package]]
name = "mio"
version = "0.6.23"
@ -943,6 +1106,15 @@ dependencies = [
"libc",
]
[[package]]
name = "object"
version = "0.27.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67ac1d3f9a1d3616fd9a60c8d74296f22406a238b6a72f5cc1e6f314df4ffbf9"
dependencies = [
"memchr",
]
[[package]]
name = "once_cell"
version = "1.8.0"
@ -1275,6 +1447,12 @@ dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "rustc-demangle"
version = "0.1.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342"
[[package]]
name = "rustc_version"
version = "0.2.3"
@ -1607,6 +1785,18 @@ dependencies = [
"unicode-xid",
]
[[package]]
name = "synstructure"
version = "0.12.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f"
dependencies = [
"proc-macro2",
"quote",
"syn",
"unicode-xid",
]
[[package]]
name = "tempfile"
version = "3.2.0"
@ -1929,8 +2119,10 @@ dependencies = [
name = "tricot"
version = "0.1.0"
dependencies = [
"accept-encoding-fork",
"acme-micro",
"anyhow",
"async-compression",
"bytes 1.1.0",
"chrono",
"envy",
@ -1941,7 +2133,6 @@ dependencies = [
"hyper 0.14.15",
"hyper-reverse-proxy",
"hyper-rustls",
"lazy_static",
"log",
"pretty_env_logger",
"rcgen",
@ -1954,7 +2145,7 @@ dependencies = [
"structopt",
"tokio 1.14.0",
"tokio-rustls",
"unicase",
"tokio-util",
]
[[package]]
@ -2262,3 +2453,32 @@ checksum = "e262a29d0e61ccf2b6190d7050d4b237535fc76ce4c1210d9caa316f71dffa75"
dependencies = [
"chrono",
]
[[package]]
name = "zstd"
version = "0.7.0+zstd.1.4.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9428752481d8372e15b1bf779ea518a179ad6c771cca2d2c60e4fbff3cc2cd52"
dependencies = [
"zstd-safe",
]
[[package]]
name = "zstd-safe"
version = "3.1.0+zstd.1.4.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5aa1926623ad7fe406e090555387daf73db555b948134b4d73eac5eb08fb666d"
dependencies = [
"libc",
"zstd-sys",
]
[[package]]
name = "zstd-sys"
version = "1.5.0+zstd.1.4.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e6c094340240369025fc6b731b054ee2a834328fa584310ac96aa4baebdc465"
dependencies = [
"cc",
"libc",
]

View file

@ -28,8 +28,9 @@ tokio-rustls = "0.23"
hyper-rustls = "0.23"
http = "0.2"
hyper-reverse-proxy = "0.4"
unicase = "2"
lazy_static = "1.4"
structopt = "0.3"
glob = "0.3"
rcgen = "0.8"
accept-encoding-fork = "0.2.0-alpha.3"
async-compression = { version = "0.3", features = ["tokio", "gzip", "zstd", "deflate", "brotli"] }
tokio-util = { version = "0.6", features = ["io"] }

View file

@ -1,44 +1,56 @@
use std::convert::Infallible;
use std::net::SocketAddr;
use std::sync::{atomic::Ordering, Arc};
use anyhow::Result;
use log::*;
use futures::FutureExt;
use accept_encoding_fork::Encoding;
use async_compression::tokio::bufread::*;
use futures::TryStreamExt;
use http::header::{HeaderName, HeaderValue};
use hyper::server::conn::Http;
use hyper::service::service_fn;
use hyper::{Body, Request, Response, StatusCode};
use hyper::{header, Body, Request, Response, StatusCode};
use tokio::net::TcpListener;
use tokio::sync::watch;
use tokio_rustls::TlsAcceptor;
use tokio_util::io::{ReaderStream, StreamReader};
use crate::cert_store::{CertStore, StoreResolver};
use crate::proxy_config::ProxyConfig;
use crate::reverse_proxy;
pub struct HttpsConfig {
pub bind_addr: SocketAddr,
pub enable_compression: bool,
pub compress_mime_types: Vec<String>,
}
pub async fn serve_https(
bind_addr: SocketAddr,
config: HttpsConfig,
cert_store: Arc<CertStore>,
proxy_config: watch::Receiver<Arc<ProxyConfig>>,
rx_proxy_config: watch::Receiver<Arc<ProxyConfig>>,
) -> Result<()> {
let mut cfg = rustls::ServerConfig::builder()
let config = Arc::new(config);
let mut tls_cfg = rustls::ServerConfig::builder()
.with_safe_defaults()
.with_no_client_auth()
.with_cert_resolver(Arc::new(StoreResolver(cert_store)));
cfg.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec()];
let tls_cfg = Arc::new(cfg);
let tls_acceptor = Arc::new(TlsAcceptor::from(tls_cfg));
tls_cfg.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec()];
let tls_acceptor = Arc::new(TlsAcceptor::from(Arc::new(tls_cfg)));
info!("Starting to serve on https://{}.", bind_addr);
info!("Starting to serve on https://{}.", config.bind_addr);
let tcp = TcpListener::bind(bind_addr).await?;
let tcp = TcpListener::bind(config.bind_addr).await?;
loop {
let (socket, remote_addr) = tcp.accept().await?;
let proxy_config = proxy_config.clone();
let rx_proxy_config = rx_proxy_config.clone();
let tls_acceptor = tls_acceptor.clone();
let config = config.clone();
tokio::spawn(async move {
match tls_acceptor.accept(socket).await {
@ -48,17 +60,10 @@ pub async fn serve_https(
.serve_connection(
stream,
service_fn(move |req: Request<Body>| {
let proxy_config: Arc<ProxyConfig> = proxy_config.borrow().clone();
handle(remote_addr, req, proxy_config).map(|res| match res {
Err(e) => {
warn!("Handler error: {}", e);
Response::builder()
.status(StatusCode::INTERNAL_SERVER_ERROR)
.body(Body::from(format!("{}", e)))
.map_err(Into::into)
}
x => x,
})
let https_config = config.clone();
let proxy_config: Arc<ProxyConfig> =
rx_proxy_config.borrow().clone();
handle_outer(remote_addr, req, https_config, proxy_config)
}),
)
.await;
@ -72,11 +77,30 @@ pub async fn serve_https(
}
}
async fn handle_outer(
remote_addr: SocketAddr,
req: Request<Body>,
https_config: Arc<HttpsConfig>,
proxy_config: Arc<ProxyConfig>,
) -> Result<Response<Body>, Infallible> {
match handle(remote_addr, req, https_config, proxy_config).await {
Err(e) => {
warn!("Handler error: {}", e);
Ok(Response::builder()
.status(StatusCode::INTERNAL_SERVER_ERROR)
.body(Body::from(format!("{}", e)))
.unwrap())
}
Ok(r) => Ok(r),
}
}
// Custom echo service, handling two different routes and a
// catch-all 404 responder.
async fn handle(
remote_addr: SocketAddr,
req: Request<Body>,
https_config: Arc<HttpsConfig>,
proxy_config: Arc<ProxyConfig>,
) -> Result<Response<Body>, anyhow::Error> {
let method = req.method().clone();
@ -91,6 +115,7 @@ async fn handle(
.to_str()?
};
let path = req.uri().path();
let accept_encoding = accept_encoding_fork::parse(req.headers()).unwrap_or(None);
let best_match = proxy_config
.entries
@ -137,7 +162,11 @@ async fn handle(
trace!("Response: {:?}", response);
info!("{} {} {}", method, response.status().as_u16(), uri);
Ok(response)
if https_config.enable_compression {
try_compress(response, accept_encoding, &https_config)
} else {
Ok(response)
}
} else {
debug!("{}{} -> NOT FOUND", host, path);
info!("{} 404 {}", method, uri);
@ -147,3 +176,61 @@ async fn handle(
.body(Body::from("No matching proxy entry"))?)
}
}
fn try_compress(
response: Response<Body>,
accept_encoding: Option<Encoding>,
https_config: &HttpsConfig,
) -> Result<Response<Body>> {
// Check if a compression encoding is accepted
let encoding = match accept_encoding {
None | Some(Encoding::Identity) => return Ok(response),
Some(enc) => enc,
};
// If already compressed, return as is
if response.headers().get(header::CONTENT_ENCODING).is_some() {
return Ok(response);
}
// If content type not in mime types for which to compress, return as is
match response.headers().get(header::CONTENT_TYPE) {
Some(ct) => {
let ct_str = ct.to_str()?;
if !https_config.compress_mime_types.iter().any(|x| x == ct_str) {
return Ok(response);
}
}
None => return Ok(response),
};
debug!("Compressing response body as {:?}", encoding);
let (mut head, body) = response.into_parts();
let body_rd =
StreamReader::new(body.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e)));
let compressed_body = match encoding {
Encoding::Gzip => {
head.headers
.insert(header::CONTENT_ENCODING, "gzip".parse()?);
Body::wrap_stream(ReaderStream::new(GzipEncoder::new(body_rd)))
}
Encoding::Brotli => {
head.headers.insert(header::CONTENT_ENCODING, "br".parse()?);
Body::wrap_stream(ReaderStream::new(BrotliEncoder::new(body_rd)))
}
Encoding::Deflate => {
head.headers
.insert(header::CONTENT_ENCODING, "deflate".parse()?);
Body::wrap_stream(ReaderStream::new(DeflateEncoder::new(body_rd)))
}
Encoding::Zstd => {
head.headers
.insert(header::CONTENT_ENCODING, "zstd".parse()?);
Body::wrap_stream(ReaderStream::new(ZstdEncoder::new(body_rd)))
}
_ => unreachable!(),
};
Ok(Response::from_parts(head, compressed_body))
}

View file

@ -58,6 +58,18 @@ struct Opt {
/// E-mail address for Let's Encrypt certificate requests
#[structopt(long = "letsencrypt-email", env = "TRICOT_LETSENCRYPT_EMAIL")]
pub letsencrypt_email: String,
/// Enable compression of responses
#[structopt(long = "enable-compression", env = "TRICOT_ENABLE_COMPRESSION")]
pub enable_compression: bool,
/// Mime types for which to enable compression (comma-separated list)
#[structopt(
long = "compress-mime-types",
env = "TRICOT_COMPRESS_MIME_TYPES",
default_value = "text/html,text/plain,text/css,text/javascript,application/javascript,image/svg+xml"
)]
pub compress_mime_types: String,
}
#[tokio::main(flavor = "multi_thread", worker_threads = 10)]
@ -87,13 +99,19 @@ async fn main() {
);
tokio::spawn(http::serve_http(opt.http_bind_addr, consul.clone()).map_err(exit_on_err));
let https_config = https::HttpsConfig {
bind_addr: opt.https_bind_addr,
enable_compression: opt.enable_compression,
compress_mime_types: opt
.compress_mime_types
.split(",")
.map(|x| x.to_string())
.collect(),
};
tokio::spawn(
https::serve_https(
opt.https_bind_addr,
cert_store.clone(),
rx_proxy_config.clone(),
)
.map_err(exit_on_err),
https::serve_https(https_config, cert_store.clone(), rx_proxy_config.clone())
.map_err(exit_on_err),
);
while rx_proxy_config.changed().await.is_ok() {

View file

@ -12,33 +12,25 @@ use log::*;
use http::header::HeaderName;
use hyper::header::{HeaderMap, HeaderValue};
use hyper::{Body, Client, Request, Response, Uri};
use lazy_static::lazy_static;
use hyper::{header, Body, Client, Request, Response, Uri};
use rustls::client::{ServerCertVerified, ServerCertVerifier};
use rustls::{Certificate, ServerName};
use crate::tls_util::HttpsConnectorFixedDnsname;
fn is_hop_header(name: &str) -> bool {
use unicase::Ascii;
const HOP_HEADERS: &[HeaderName] = &[
header::CONNECTION,
//header::KEEP_ALIVE,
header::PROXY_AUTHENTICATE,
header::PROXY_AUTHORIZATION,
header::TE,
header::TRAILER,
header::TRANSFER_ENCODING,
header::UPGRADE,
];
// A list of the headers, using `unicase` to help us compare without
// worrying about the case, and `lazy_static!` to prevent reallocation
// of the vector.
lazy_static! {
static ref HOP_HEADERS: Vec<Ascii<&'static str>> = vec![
Ascii::new("Connection"),
Ascii::new("Keep-Alive"),
Ascii::new("Proxy-Authenticate"),
Ascii::new("Proxy-Authorization"),
Ascii::new("Te"),
Ascii::new("Trailers"),
Ascii::new("Transfer-Encoding"),
Ascii::new("Upgrade"),
];
}
HOP_HEADERS.iter().any(|h| h == &name)
fn is_hop_header(name: &HeaderName) -> bool {
HOP_HEADERS.iter().any(|h| h == name)
}
/// Returns a clone of the headers without the [hop-by-hop headers].
@ -47,7 +39,7 @@ fn is_hop_header(name: &str) -> bool {
fn remove_hop_headers(headers: &HeaderMap<HeaderValue>) -> HeaderMap<HeaderValue> {
let mut result = HeaderMap::new();
for (k, v) in headers.iter() {
if !is_hop_header(k.as_str()) {
if !is_hop_header(&k) {
result.append(k.clone(), v.clone());
}
}
@ -80,10 +72,7 @@ fn create_proxied_request<B>(
*builder.headers_mut().unwrap() = remove_hop_headers(request.headers());
// If request does not have host header, add it from original URI authority
let host_header_name = "host";
if let hyper::header::Entry::Vacant(entry) =
builder.headers_mut().unwrap().entry(host_header_name)
{
if let header::Entry::Vacant(entry) = builder.headers_mut().unwrap().entry(header::HOST) {
if let Some(authority) = request.uri().authority() {
entry.insert(authority.as_str().parse()?);
}
@ -96,11 +85,11 @@ fn create_proxied_request<B>(
.unwrap()
.entry(x_forwarded_for_header_name)
{
hyper::header::Entry::Vacant(entry) => {
header::Entry::Vacant(entry) => {
entry.insert(client_ip.to_string().parse()?);
}
hyper::header::Entry::Occupied(mut entry) => {
header::Entry::Occupied(mut entry) => {
let addr = format!("{}, {}", entry.get().to_str()?, client_ip);
entry.insert(addr.parse()?);
}
@ -112,17 +101,17 @@ fn create_proxied_request<B>(
);
// Proxy upgrade requests properly
if let Some(conn) = request.headers().get("connection") {
if let Some(conn) = request.headers().get(header::CONNECTION) {
if conn.to_str()?.to_lowercase() == "upgrade" {
if let Some(upgrade) = request.headers().get("upgrade") {
builder.headers_mut().unwrap().insert(
HeaderName::from_bytes(b"connection")?,
"Upgrade".try_into()?,
);
if let Some(upgrade) = request.headers().get(header::UPGRADE) {
builder
.headers_mut()
.unwrap()
.insert(HeaderName::from_bytes(b"upgrade")?, upgrade.clone());
.insert(header::CONNECTION, "Upgrade".try_into()?);
builder
.headers_mut()
.unwrap()
.insert(header::UPGRADE, upgrade.clone());
}
}
}