Compare commits
No commits in common. "main" and "quentin" have entirely different histories.
|
|
@ -1,4 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euxo pipefail
|
||||
|
||||
nix build --print-build-logs .#packages.x86_64-unknown-linux-musl.debug
|
||||
2
.gitattributes
vendored
2
.gitattributes
vendored
|
|
@ -1,2 +0,0 @@
|
|||
Cargo.nix linguist-vendored
|
||||
flake.lock linguist-vendored
|
||||
2
.gitignore
vendored
2
.gitignore
vendored
|
|
@ -1,6 +1,6 @@
|
|||
/target
|
||||
.vimrc
|
||||
env.sh
|
||||
aerogramme.toml
|
||||
mailrage.toml
|
||||
*.swo
|
||||
*.swp
|
||||
|
|
|
|||
3384
Cargo.lock
generated
3384
Cargo.lock
generated
File diff suppressed because it is too large
Load diff
104
Cargo.toml
104
Cargo.toml
|
|
@ -1,82 +1,48 @@
|
|||
[package]
|
||||
name = "aerogramme"
|
||||
version = "0.2.2"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"]
|
||||
name = "mailrage"
|
||||
version = "0.0.1"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2021"
|
||||
license = "EUPL-1.2"
|
||||
description = "A robust email server"
|
||||
license = "AGPL-3.0"
|
||||
description = "Encrypted mail storage over Garage"
|
||||
|
||||
[dependencies]
|
||||
# async runtime
|
||||
anyhow = "1.0.28"
|
||||
argon2 = "0.3"
|
||||
async-trait = "0.1"
|
||||
base64 = "0.13"
|
||||
clap = { version = "3.1.18", features = ["derive", "env"] }
|
||||
duplexify = "1.1.0"
|
||||
hex = "0.4"
|
||||
futures = "0.3"
|
||||
im = "15"
|
||||
itertools = "0.10"
|
||||
lazy_static = "1.4"
|
||||
ldap3 = { version = "0.10", default-features = false, features = ["tls"] }
|
||||
log = "0.4"
|
||||
pretty_env_logger = "0.4"
|
||||
rusoto_core = "0.48.0"
|
||||
rusoto_credential = "0.48.0"
|
||||
rusoto_s3 = "0.48"
|
||||
rusoto_signature = "0.48.0"
|
||||
serde = "1.0.137"
|
||||
rand = "0.8.5"
|
||||
rmp-serde = "0.15"
|
||||
rpassword = "6.0"
|
||||
sodiumoxide = "0.2"
|
||||
tokio = { version = "1.18", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
|
||||
tokio-util = { version = "0.7", features = [ "compat" ] }
|
||||
futures = "0.3"
|
||||
|
||||
# debug
|
||||
log = "0.4"
|
||||
backtrace = "0.3"
|
||||
console-subscriber = "0.2"
|
||||
tracing-subscriber = "0.3"
|
||||
tracing = "0.1"
|
||||
|
||||
# language extensions
|
||||
lazy_static = "1.4"
|
||||
duplexify = "1.1.0"
|
||||
im = "15"
|
||||
anyhow = "1.0.28"
|
||||
async-trait = "0.1"
|
||||
itertools = "0.10"
|
||||
chrono = { version = "0.4", default-features = false, features = ["alloc"] }
|
||||
|
||||
# process related
|
||||
nix = { version = "0.27", features = ["signal"] }
|
||||
clap = { version = "3.1.18", features = ["derive", "env"] }
|
||||
|
||||
# serialization & parsing
|
||||
serde = "1.0.137"
|
||||
rmp-serde = "0.15"
|
||||
toml = "0.5"
|
||||
base64 = "0.21"
|
||||
hex = "0.4"
|
||||
nom = "7.1"
|
||||
zstd = { version = "0.9", default-features = false }
|
||||
|
||||
# cryptography & security
|
||||
sodiumoxide = "0.2"
|
||||
argon2 = "0.5"
|
||||
rand = "0.8.5"
|
||||
rustls = "0.22"
|
||||
rustls-pemfile = "2.0"
|
||||
tokio-rustls = "0.25"
|
||||
hyper-rustls = { version = "0.26", features = ["http2"] }
|
||||
hyper-util = { version = "0.1", features = ["full"] }
|
||||
rpassword = "7.0"
|
||||
tracing-subscriber = "0.3"
|
||||
tracing = "0.1"
|
||||
tower = "0.4"
|
||||
imap-codec = "0.5"
|
||||
|
||||
# login
|
||||
ldap3 = { version = "0.10", default-features = false, features = ["tls-rustls"] }
|
||||
|
||||
# storage
|
||||
k2v-client = { git = "https://git.deuxfleurs.fr/Deuxfleurs/garage.git", branch = "k2v/shared_http_client" }
|
||||
aws-config = { version = "1", features = ["behavior-version-latest"] }
|
||||
aws-sdk-s3 = "1"
|
||||
aws-smithy-runtime = "1"
|
||||
aws-smithy-runtime-api = "1"
|
||||
|
||||
# email protocols
|
||||
eml-codec = "0.1.2"
|
||||
k2v-client = { git = "https://git.deuxfleurs.fr/Deuxfleurs/garage.git", branch = "main" }
|
||||
boitalettres = { git = "https://git.deuxfleurs.fr/KokaKiwi/boitalettres.git", branch = "main" }
|
||||
smtp-message = { git = "http://github.com/Alexis211/kannader", branch = "feature/lmtp" }
|
||||
smtp-server = { git = "http://github.com/Alexis211/kannader", branch = "feature/lmtp" }
|
||||
imap-codec = { version = "2.0.0", features = ["bounded-static", "ext_condstore_qresync"] }
|
||||
imap-flow = { git = "https://github.com/duesee/imap-flow.git", branch = "main" }
|
||||
thiserror = "1.0.56"
|
||||
|
||||
[dev-dependencies]
|
||||
|
||||
[patch.crates-io]
|
||||
imap-types = { git = "https://github.com/superboum/imap-codec", branch = "custom/aerogramme" }
|
||||
imap-codec = { git = "https://github.com/superboum/imap-codec", branch = "custom/aerogramme" }
|
||||
|
||||
[[test]]
|
||||
name = "behavior"
|
||||
path = "tests/behavior.rs"
|
||||
harness = false
|
||||
#k2v-client = { path = "../garage/src/k2v-client" }
|
||||
|
|
|
|||
287
LICENSE
287
LICENSE
|
|
@ -1,287 +0,0 @@
|
|||
EUROPEAN UNION PUBLIC LICENCE v. 1.2
|
||||
EUPL © the European Union 2007, 2016
|
||||
|
||||
This European Union Public Licence (the ‘EUPL’) applies to the Work (as defined
|
||||
below) which is provided under the terms of this Licence. Any use of the Work,
|
||||
other than as authorised under this Licence is prohibited (to the extent such
|
||||
use is covered by a right of the copyright holder of the Work).
|
||||
|
||||
The Work is provided under the terms of this Licence when the Licensor (as
|
||||
defined below) has placed the following notice immediately following the
|
||||
copyright notice for the Work:
|
||||
|
||||
Licensed under the EUPL
|
||||
|
||||
or has expressed by any other means his willingness to license under the EUPL.
|
||||
|
||||
1. Definitions
|
||||
|
||||
In this Licence, the following terms have the following meaning:
|
||||
|
||||
- ‘The Licence’: this Licence.
|
||||
|
||||
- ‘The Original Work’: the work or software distributed or communicated by the
|
||||
Licensor under this Licence, available as Source Code and also as Executable
|
||||
Code as the case may be.
|
||||
|
||||
- ‘Derivative Works’: the works or software that could be created by the
|
||||
Licensee, based upon the Original Work or modifications thereof. This Licence
|
||||
does not define the extent of modification or dependence on the Original Work
|
||||
required in order to classify a work as a Derivative Work; this extent is
|
||||
determined by copyright law applicable in the country mentioned in Article 15.
|
||||
|
||||
- ‘The Work’: the Original Work or its Derivative Works.
|
||||
|
||||
- ‘The Source Code’: the human-readable form of the Work which is the most
|
||||
convenient for people to study and modify.
|
||||
|
||||
- ‘The Executable Code’: any code which has generally been compiled and which is
|
||||
meant to be interpreted by a computer as a program.
|
||||
|
||||
- ‘The Licensor’: the natural or legal person that distributes or communicates
|
||||
the Work under the Licence.
|
||||
|
||||
- ‘Contributor(s)’: any natural or legal person who modifies the Work under the
|
||||
Licence, or otherwise contributes to the creation of a Derivative Work.
|
||||
|
||||
- ‘The Licensee’ or ‘You’: any natural or legal person who makes any usage of
|
||||
the Work under the terms of the Licence.
|
||||
|
||||
- ‘Distribution’ or ‘Communication’: any act of selling, giving, lending,
|
||||
renting, distributing, communicating, transmitting, or otherwise making
|
||||
available, online or offline, copies of the Work or providing access to its
|
||||
essential functionalities at the disposal of any other natural or legal
|
||||
person.
|
||||
|
||||
2. Scope of the rights granted by the Licence
|
||||
|
||||
The Licensor hereby grants You a worldwide, royalty-free, non-exclusive,
|
||||
sublicensable licence to do the following, for the duration of copyright vested
|
||||
in the Original Work:
|
||||
|
||||
- use the Work in any circumstance and for all usage,
|
||||
- reproduce the Work,
|
||||
- modify the Work, and make Derivative Works based upon the Work,
|
||||
- communicate to the public, including the right to make available or display
|
||||
the Work or copies thereof to the public and perform publicly, as the case may
|
||||
be, the Work,
|
||||
- distribute the Work or copies thereof,
|
||||
- lend and rent the Work or copies thereof,
|
||||
- sublicense rights in the Work or copies thereof.
|
||||
|
||||
Those rights can be exercised on any media, supports and formats, whether now
|
||||
known or later invented, as far as the applicable law permits so.
|
||||
|
||||
In the countries where moral rights apply, the Licensor waives his right to
|
||||
exercise his moral right to the extent allowed by law in order to make effective
|
||||
the licence of the economic rights here above listed.
|
||||
|
||||
The Licensor grants to the Licensee royalty-free, non-exclusive usage rights to
|
||||
any patents held by the Licensor, to the extent necessary to make use of the
|
||||
rights granted on the Work under this Licence.
|
||||
|
||||
3. Communication of the Source Code
|
||||
|
||||
The Licensor may provide the Work either in its Source Code form, or as
|
||||
Executable Code. If the Work is provided as Executable Code, the Licensor
|
||||
provides in addition a machine-readable copy of the Source Code of the Work
|
||||
along with each copy of the Work that the Licensor distributes or indicates, in
|
||||
a notice following the copyright notice attached to the Work, a repository where
|
||||
the Source Code is easily and freely accessible for as long as the Licensor
|
||||
continues to distribute or communicate the Work.
|
||||
|
||||
4. Limitations on copyright
|
||||
|
||||
Nothing in this Licence is intended to deprive the Licensee of the benefits from
|
||||
any exception or limitation to the exclusive rights of the rights owners in the
|
||||
Work, of the exhaustion of those rights or of other applicable limitations
|
||||
thereto.
|
||||
|
||||
5. Obligations of the Licensee
|
||||
|
||||
The grant of the rights mentioned above is subject to some restrictions and
|
||||
obligations imposed on the Licensee. Those obligations are the following:
|
||||
|
||||
Attribution right: The Licensee shall keep intact all copyright, patent or
|
||||
trademarks notices and all notices that refer to the Licence and to the
|
||||
disclaimer of warranties. The Licensee must include a copy of such notices and a
|
||||
copy of the Licence with every copy of the Work he/she distributes or
|
||||
communicates. The Licensee must cause any Derivative Work to carry prominent
|
||||
notices stating that the Work has been modified and the date of modification.
|
||||
|
||||
Copyleft clause: If the Licensee distributes or communicates copies of the
|
||||
Original Works or Derivative Works, this Distribution or Communication will be
|
||||
done under the terms of this Licence or of a later version of this Licence
|
||||
unless the Original Work is expressly distributed only under this version of the
|
||||
Licence — for example by communicating ‘EUPL v. 1.2 only’. The Licensee
|
||||
(becoming Licensor) cannot offer or impose any additional terms or conditions on
|
||||
the Work or Derivative Work that alter or restrict the terms of the Licence.
|
||||
|
||||
Compatibility clause: If the Licensee Distributes or Communicates Derivative
|
||||
Works or copies thereof based upon both the Work and another work licensed under
|
||||
a Compatible Licence, this Distribution or Communication can be done under the
|
||||
terms of this Compatible Licence. For the sake of this clause, ‘Compatible
|
||||
Licence’ refers to the licences listed in the appendix attached to this Licence.
|
||||
Should the Licensee's obligations under the Compatible Licence conflict with
|
||||
his/her obligations under this Licence, the obligations of the Compatible
|
||||
Licence shall prevail.
|
||||
|
||||
Provision of Source Code: When distributing or communicating copies of the Work,
|
||||
the Licensee will provide a machine-readable copy of the Source Code or indicate
|
||||
a repository where this Source will be easily and freely available for as long
|
||||
as the Licensee continues to distribute or communicate the Work.
|
||||
|
||||
Legal Protection: This Licence does not grant permission to use the trade names,
|
||||
trademarks, service marks, or names of the Licensor, except as required for
|
||||
reasonable and customary use in describing the origin of the Work and
|
||||
reproducing the content of the copyright notice.
|
||||
|
||||
6. Chain of Authorship
|
||||
|
||||
The original Licensor warrants that the copyright in the Original Work granted
|
||||
hereunder is owned by him/her or licensed to him/her and that he/she has the
|
||||
power and authority to grant the Licence.
|
||||
|
||||
Each Contributor warrants that the copyright in the modifications he/she brings
|
||||
to the Work are owned by him/her or licensed to him/her and that he/she has the
|
||||
power and authority to grant the Licence.
|
||||
|
||||
Each time You accept the Licence, the original Licensor and subsequent
|
||||
Contributors grant You a licence to their contributions to the Work, under the
|
||||
terms of this Licence.
|
||||
|
||||
7. Disclaimer of Warranty
|
||||
|
||||
The Work is a work in progress, which is continuously improved by numerous
|
||||
Contributors. It is not a finished work and may therefore contain defects or
|
||||
‘bugs’ inherent to this type of development.
|
||||
|
||||
For the above reason, the Work is provided under the Licence on an ‘as is’ basis
|
||||
and without warranties of any kind concerning the Work, including without
|
||||
limitation merchantability, fitness for a particular purpose, absence of defects
|
||||
or errors, accuracy, non-infringement of intellectual property rights other than
|
||||
copyright as stated in Article 6 of this Licence.
|
||||
|
||||
This disclaimer of warranty is an essential part of the Licence and a condition
|
||||
for the grant of any rights to the Work.
|
||||
|
||||
8. Disclaimer of Liability
|
||||
|
||||
Except in the cases of wilful misconduct or damages directly caused to natural
|
||||
persons, the Licensor will in no event be liable for any direct or indirect,
|
||||
material or moral, damages of any kind, arising out of the Licence or of the use
|
||||
of the Work, including without limitation, damages for loss of goodwill, work
|
||||
stoppage, computer failure or malfunction, loss of data or any commercial
|
||||
damage, even if the Licensor has been advised of the possibility of such damage.
|
||||
However, the Licensor will be liable under statutory product liability laws as
|
||||
far such laws apply to the Work.
|
||||
|
||||
9. Additional agreements
|
||||
|
||||
While distributing the Work, You may choose to conclude an additional agreement,
|
||||
defining obligations or services consistent with this Licence. However, if
|
||||
accepting obligations, You may act only on your own behalf and on your sole
|
||||
responsibility, not on behalf of the original Licensor or any other Contributor,
|
||||
and only if You agree to indemnify, defend, and hold each Contributor harmless
|
||||
for any liability incurred by, or claims asserted against such Contributor by
|
||||
the fact You have accepted any warranty or additional liability.
|
||||
|
||||
10. Acceptance of the Licence
|
||||
|
||||
The provisions of this Licence can be accepted by clicking on an icon ‘I agree’
|
||||
placed under the bottom of a window displaying the text of this Licence or by
|
||||
affirming consent in any other similar way, in accordance with the rules of
|
||||
applicable law. Clicking on that icon indicates your clear and irrevocable
|
||||
acceptance of this Licence and all of its terms and conditions.
|
||||
|
||||
Similarly, you irrevocably accept this Licence and all of its terms and
|
||||
conditions by exercising any rights granted to You by Article 2 of this Licence,
|
||||
such as the use of the Work, the creation by You of a Derivative Work or the
|
||||
Distribution or Communication by You of the Work or copies thereof.
|
||||
|
||||
11. Information to the public
|
||||
|
||||
In case of any Distribution or Communication of the Work by means of electronic
|
||||
communication by You (for example, by offering to download the Work from a
|
||||
remote location) the distribution channel or media (for example, a website) must
|
||||
at least provide to the public the information requested by the applicable law
|
||||
regarding the Licensor, the Licence and the way it may be accessible, concluded,
|
||||
stored and reproduced by the Licensee.
|
||||
|
||||
12. Termination of the Licence
|
||||
|
||||
The Licence and the rights granted hereunder will terminate automatically upon
|
||||
any breach by the Licensee of the terms of the Licence.
|
||||
|
||||
Such a termination will not terminate the licences of any person who has
|
||||
received the Work from the Licensee under the Licence, provided such persons
|
||||
remain in full compliance with the Licence.
|
||||
|
||||
13. Miscellaneous
|
||||
|
||||
Without prejudice of Article 9 above, the Licence represents the complete
|
||||
agreement between the Parties as to the Work.
|
||||
|
||||
If any provision of the Licence is invalid or unenforceable under applicable
|
||||
law, this will not affect the validity or enforceability of the Licence as a
|
||||
whole. Such provision will be construed or reformed so as necessary to make it
|
||||
valid and enforceable.
|
||||
|
||||
The European Commission may publish other linguistic versions or new versions of
|
||||
this Licence or updated versions of the Appendix, so far this is required and
|
||||
reasonable, without reducing the scope of the rights granted by the Licence. New
|
||||
versions of the Licence will be published with a unique version number.
|
||||
|
||||
All linguistic versions of this Licence, approved by the European Commission,
|
||||
have identical value. Parties can take advantage of the linguistic version of
|
||||
their choice.
|
||||
|
||||
14. Jurisdiction
|
||||
|
||||
Without prejudice to specific agreement between parties,
|
||||
|
||||
- any litigation resulting from the interpretation of this License, arising
|
||||
between the European Union institutions, bodies, offices or agencies, as a
|
||||
Licensor, and any Licensee, will be subject to the jurisdiction of the Court
|
||||
of Justice of the European Union, as laid down in article 272 of the Treaty on
|
||||
the Functioning of the European Union,
|
||||
|
||||
- any litigation arising between other parties and resulting from the
|
||||
interpretation of this License, will be subject to the exclusive jurisdiction
|
||||
of the competent court where the Licensor resides or conducts its primary
|
||||
business.
|
||||
|
||||
15. Applicable Law
|
||||
|
||||
Without prejudice to specific agreement between parties,
|
||||
|
||||
- this Licence shall be governed by the law of the European Union Member State
|
||||
where the Licensor has his seat, resides or has his registered office,
|
||||
|
||||
- this licence shall be governed by Belgian law if the Licensor has no seat,
|
||||
residence or registered office inside a European Union Member State.
|
||||
|
||||
Appendix
|
||||
|
||||
‘Compatible Licences’ according to Article 5 EUPL are:
|
||||
|
||||
- GNU General Public License (GPL) v. 2, v. 3
|
||||
- GNU Affero General Public License (AGPL) v. 3
|
||||
- Open Software License (OSL) v. 2.1, v. 3.0
|
||||
- Eclipse Public License (EPL) v. 1.0
|
||||
- CeCILL v. 2.0, v. 2.1
|
||||
- Mozilla Public Licence (MPL) v. 2
|
||||
- GNU Lesser General Public Licence (LGPL) v. 2.1, v. 3
|
||||
- Creative Commons Attribution-ShareAlike v. 3.0 Unported (CC BY-SA 3.0) for
|
||||
works other than software
|
||||
- European Union Public Licence (EUPL) v. 1.1, v. 1.2
|
||||
- Québec Free and Open-Source Licence — Reciprocity (LiLiQ-R) or Strong
|
||||
Reciprocity (LiLiQ-R+).
|
||||
|
||||
The European Commission may update this Appendix to later versions of the above
|
||||
licences without producing a new version of the EUPL, as long as they provide
|
||||
the rights granted in Article 2 of this Licence and protect the covered Source
|
||||
Code from exclusive appropriation.
|
||||
|
||||
All other changes or additions to this Appendix require the production of a new
|
||||
EUPL version.
|
||||
216
README.md
216
README.md
|
|
@ -1,57 +1,199 @@
|
|||
|
||||
# Mailrage - Encrypted e-mail storage over Garage
|
||||
|
||||
# Aerogramme - Encrypted e-mail storage over Garage
|
||||
## Usage
|
||||
|
||||
⚠️ **TECHNOLOGICAL PREVIEW, THIS SERVER IS NOT READY FOR PRODUCTION OR EVEN BETA TESTING**
|
||||
Start by running:
|
||||
|
||||
A resilient & standards-compliant open-source IMAP server with built-in encryption
|
||||
```
|
||||
$ cargo run --bin main -- first-login --region garage --k2v-endpoint http://127.0.0.1:3904 --s3-endpoint http://127.0.0.1:3900 --aws-access-key-id GK... --aws-secret-access-key c0ffee... --bucket mailrage-quentin --user-secret poupou
|
||||
Please enter your password for key decryption.
|
||||
If you are using LDAP login, this must be your LDAP password.
|
||||
If you are using the static login provider, enter any password, and this will also become your password for local IMAP access.
|
||||
Enter password:
|
||||
Confirm password:
|
||||
|
||||
## Quickly jump to our website!
|
||||
Cryptographic key setup is complete.
|
||||
|
||||
<a href="https://aerogramme.deuxfleurs.fr/download/"><img height="100" src="https://aerogramme.deuxfleurs.fr/images/download.png" alt="Download"/></a>
|
||||
<a href="https://aerogramme.deuxfleurs.fr/documentation/quick-start/"><img height="100" src="https://aerogramme.deuxfleurs.fr/images/getting-started.png" alt="Getting Start"/></a>
|
||||
If you are using the static login provider, add the following section to your .toml configuration file:
|
||||
|
||||
[RFC Coverage](https://aerogramme.deuxfleurs.fr/documentation/reference/rfc/) -
|
||||
[Design overview](https://aerogramme.deuxfleurs.fr/documentation/design/overview/) -
|
||||
[Mailbox Datastructure](https://aerogramme.deuxfleurs.fr/documentation/design/mailbox/) -
|
||||
[Mailbox Mutation Log](https://aerogramme.deuxfleurs.fr/documentation/design/log/).
|
||||
[login_static.users.<username>]
|
||||
password = "$argon2id$v=19$m=4096,t=3,p=1$..."
|
||||
aws_access_key_id = "GK..."
|
||||
aws_secret_access_key = "c0ffee..."
|
||||
```
|
||||
|
||||
## Roadmap
|
||||
Next create the config file `mailrage.toml`:
|
||||
|
||||
- ✅ 0.1 Better emails parsing (july '23, see [eml-codec](https://git.deuxfleurs.fr/Deuxfleurs/eml-codec)).
|
||||
- ✅ 0.2 Support of IMAP4. (~january '24).
|
||||
- ⌛0.3 CalDAV support. (~february '24).
|
||||
- ⌛0.4 CardDAV support.
|
||||
- ⌛0.5 Public beta.
|
||||
```
|
||||
s3_endpoint = "http://127.0.0.1:3900"
|
||||
k2v_endpoint = "http://127.0.0.1:3904"
|
||||
aws_region = "garage"
|
||||
|
||||
## A note about cargo2nix
|
||||
[login_static]
|
||||
default_bucket = "mailrage"
|
||||
[login_static.users.quentin]
|
||||
bucket = "mailrage-quentin"
|
||||
user_secret = "poupou"
|
||||
alternate_user_secrets = []
|
||||
password = "$argon2id$v=19$m=4096,t=3,p=1$..."
|
||||
aws_access_key_id = "GK..."
|
||||
aws_secret_access_key = "c0ffee..."
|
||||
```
|
||||
|
||||
Currently, you must edit Cargo.nix by hand after running `cargo2nix`.
|
||||
Find the `tokio` dependency declaration.
|
||||
Look at tokio's dependencies, the `tracing` is disable through a `if false` logic.
|
||||
Activate it by replacing the condition with `if true`.
|
||||
You can dump your keys with:
|
||||
|
||||
```
|
||||
$ cargo run --bin main -- show-keys --region garage --k2v-endpoint http://127.0.0.1:3904 --s3-endpoint http://127.0.0.1:3900 --aws-access-key-id GK... --aws-secret-access-key c0ffee... --bucket mailrage-quentin --user-secret poupou
|
||||
Enter key decryption password:
|
||||
master_key = "..."
|
||||
secret_key = "..."
|
||||
```
|
||||
|
||||
Run a test instance with:
|
||||
|
||||
```
|
||||
$ cargo run --bin main -- server
|
||||
---- MAILBOX STATE ----
|
||||
UIDVALIDITY 1
|
||||
UIDNEXT 2
|
||||
INTERNALSEQ 2
|
||||
1 c3d4524f557f19108480063f3216afa20000000000000000 \Unseen
|
||||
|
||||
---- MAILBOX STATE ----
|
||||
UIDVALIDITY 1
|
||||
UIDNEXT 3
|
||||
INTERNALSEQ 3
|
||||
1 c3d4524f557f19108480063f3216afa20000000000000000 \Unseen
|
||||
2 6a1ab4d87af3d424a3a8f8720c4db3b60000000000000000 \Unseen
|
||||
```
|
||||
|
||||
|
||||
## Sponsors and funding
|
||||
## Bayou storage module
|
||||
|
||||
[Aerogramme project](https://nlnet.nl/project/Aerogramme/) is funded through the NGI Assure Fund, a fund established by NLnet with financial support from the European Commission's Next Generation Internet programme, under the aegis of DG Communications Networks, Content and Technology under grant agreement No 957073.
|
||||
Checkpoints are stored in S3 at `<path>/checkpoint/<timestamp>`. Example:
|
||||
|
||||
|
||||
```
|
||||
348 TestMailbox/checkpoint/00000180d77400dc126b16aac546b769
|
||||
369 TestMailbox/checkpoint/00000180d776e509b68fdc5c376d0abc
|
||||
357 TestMailbox/checkpoint/00000180d77a7fe68f4f76e3b45aa751
|
||||
```
|
||||
|
||||
## License
|
||||
Operations are stored in K2V at PK `<path>`, SK `<timestamp>`. Example:
|
||||
|
||||
EUROPEAN UNION PUBLIC LICENCE v. 1.2
|
||||
EUPL © the European Union 2007, 2016
|
||||
```
|
||||
TestMailbox 00000180d77400dc126b16aac546b769 RcIsESv7WrjMuHwyI/dvCnkIfy6op5Tiylf0WSnn94aMS2uagl7YeMBwdv09TiSXBpu5nJ5e/9QFSfuEI/NqKrdQkX54MOsnaIGhRb0oqUG3KNaar3BiVSvYvXuzYhk4ii+TUS2Eyd6fCCaNVNM5
|
||||
TestMailbox 00000180d775f27f5542a13fc21c665e RrTSOup/zO1Ei+QrjBcDLt4vvFSY+WJPBodwY64wy2ftW+Oh3VSArvlO4SAEPmdsx1gt0HPBZYR/OkVWsZpmix1ZLFUmvdib+rjNkorHQW1p+oLVK8tolGrqk4SRwl88cqu466T4vBEpDu7tRbH0
|
||||
TestMailbox 00000180d775f292b3c8da00718389b4 VAwd8SRycIwsipZW5AcSG+EIYZVWn/Uj/TADbWhb4x5LVMceiRBHWVquY08RgT/lJKdhIcUqBA15bVG3klIg8tLsWJVG784NbsZwdGRczWmngcA=
|
||||
TestMailbox 00000180d775f29d24842cf375d679e0 /FbXtEwm/bijtvOdqM1XFvKUalQFAOPHp+vF9jZThZn/viY5a6W1PyHeI8kTusF6EsVPAwPHpQyjIv/ghskC0f+zUEsSUhDwQANdwLNqDLAvTA==
|
||||
TestMailbox 00000180d7768ab1dc01ff504e887c62 W/fF0WitpxJ05yHeOv96BlpGymT1kVOjkIW00t9e6UE7mxkvNflu9cZSCd8PDJd2ymC0sC9bLVFAXKmNZsmCFEEHMQSyrX61qTYo4KFCZMp5zm6fXubaYuurrzjXzfUP/R7kBvICFZlF0daf0SwX
|
||||
TestMailbox 00000180d7768aba629c7ad6adf25228 IPzYGNsSepCX2AEnee/1Eas9a3c5esPSmrNkvaj4XcFb6Ft2KC8N6ubUR3wB+K0oYCTQym6nhHG5dlAxf6NRu7Rk8YtBTBmSqtGqd6kMZ3bU5b8=
|
||||
TestMailbox 00000180d7768ac1870cda61784114d4 aaLiaWxfx1mxh6aoKE3xUUfZWhivZ/K7ixabflFDW7FO/qbpvCaa+Y6w4lQemTy6m+leAhXGN+Dbyv2qP20yJ9O4oJF5d3Lz5Iv5uF18OxhVZzw=
|
||||
TestMailbox 00000180d776e4fb294ccdab2612b406 EtUPrLgEeOyab2QRnSie4I3Me9dDh10UdwWnUKdGa/8ezMJDtiy7XlW+tUfJdqtu6Vj7nduT0emDOXbBZsNwlcmzgYNwuNu3I9AfhZTFWtwLgB+wnAgB/jim82DDrJfLia8kB2eA2ao5jfJ3uMSZ
|
||||
TestMailbox 00000180d776e501528546d340490291 Lz4Z9wCTk1lZ86lL01urhAan4oHcr1NBqdRe+CDpA51D9IncA5+Fhc8I6knUIh2qQ5/woWgISLAVwzSS+0+TxrYoqxf5FumIQtUJfwDER5La3n0=
|
||||
TestMailbox 00000180d776e509b68fdc5c376d0abc RUGE2xB3fFX/wRH/p2fHIUa+rMaXSRd7fY9zglw0pRfVPqJfpniOjAe4GHIwGlwbwjtFOwS5a+Q7yr0Wez6QwD+ohhqRFKpbjcFcN7VfMyVAf+k=
|
||||
TestMailbox 00000180d7784b987a8ad8106dc400c9 K+0LVEtBbTnWNS67jy9DtTvQyd5arovduvu490tLOE2TzVhuVoF4pfvTMTN12bH3KwEAHeDfuwKkKJFqldOywouTYPzEjZFkJzyagHrkl6dfnE5CqmlDv+Vc5TOQRskxjW+wQiZdjU8wGiBiBGYh
|
||||
TestMailbox 00000180d7784bede69ac3cff2c6b724 XMFY3+b1r1//uolVz80JSI3g/84XCk3Tm7/S0BFv+Qe/Xv3/poLrOvAKEe+GzD2s22j8p/T2RXR/JSZckzgjEZeO0wbPDXVQd94di2Pff7jxAH8=
|
||||
TestMailbox 00000180d7784bffe2595abe7ed81858 QQZhF+7wSHfikoAp93a+UY/XDIX7TVnnVYOtmQ2XHnDKA2F6snRJCPbYBO4IRHCRfVrjDGi32c41it2C3Mu5PBepabxapsW1rfIV3rlX2lkKHtI=
|
||||
TestMailbox 00000180d77a7fb3f01dbb147c20cf7f IHOlOa1JI11RUKVvQUq3HQPxiRr4UCeE+pHmL8DtNMkOh62V4spuP0VvvQTJCQcPQ1EQR/QcxZ3s7uHLkrZAHF30BkpUkGqsLBWpnyug/puhdiixWsMyLLb6G90zFjiComUwptnDc/CCXtGEHdSW
|
||||
TestMailbox 00000180d77a7fbb54b100f521ceb347 Ze4KyyTCgrYbZlXlJSY5hNob8sMXvBAmwIx2cADbX5P0M1IHXwXfloEzvvd6WYOtatFC2GnDSrmQ6RdCfeZ3WV9TZilqa0Fv0XEg48sVyVCcguw=
|
||||
TestMailbox 00000180d77a7fe68f4f76e3b45aa751 cJJVvvRzTVNKUaIHPCCDY2uY7/HlmkxGgo3ozWBlBSRDeBqU65zgZD3QIPCxa6xaqB/Gc0bQ9BGzfU0cvVmO5jgNeeDnbqqs3oeA2jml/Qv2YO9upApfNQtDT1GiwJ8vrgaIow==
|
||||
TestMailbox 00000180d8e513d3ea58c679a13178ac Ce5su2YOxNmTzk2dK8SX8V/Uue5uAC7oklEjhesY9wCMqGphhOkdWjzCqq0xOzcb/ZzzZ58t+mTksNSYIU4kddHIHBFPgqIwKthVk2mlUdqYiN/Y2vEGqv+YmtKY+GST/7Ee87ZHpU/5sv0GoXxT
|
||||
TestMailbox 00000180d8e5145a23f8faee86283900 sp3D8xFZcM9icNlDJXIUDJb3mo6VGD9f1aDHD+4RbPdx6mTYF+qNTsPHKCxHHxT/9NfNe8XPg2+8xYRtm7SXfgERZBDB8ye+Xt3fM1k+wbL6RsaJmDHVECeXeL5KHuITzpI22A==
|
||||
TestMailbox 00000180d8e51465c38f0585f9bb760e FF0VId2O/bBNzYD5ABWReMs5hHoHwynOoJRKj9vyaUMZ3JykInFmvvRgtCbJBDjTQPwPU8apphKQfwuicO76H7GtZqH009Cbv5l8ZTRJKrmzOQmtjzBQc2eGEUMPfbml5t0GCg==
|
||||
```
|
||||
|
||||
This European Union Public Licence (the ‘EUPL’) applies to the Work (as defined
|
||||
below) which is provided under the terms of this Licence. Any use of the Work,
|
||||
other than as authorised under this Licence is prohibited (to the extent such
|
||||
use is covered by a right of the copyright holder of the Work).
|
||||
The timestamp of a checkpoint corresponds to the timestamp of the first operation NOT included in the checkpoint.
|
||||
In other words, to reconstruct the final state:
|
||||
|
||||
The Work is provided under the terms of this Licence when the Licensor (as
|
||||
defined below) has placed the following notice immediately following the
|
||||
copyright notice for the Work:
|
||||
- find timestamp `<ts>` of last checkpoint
|
||||
- load checkpoint `<ts>`
|
||||
- load and apply all operations starting from `<ts>`, included
|
||||
|
||||
Licensed under the EUPL
|
||||
## UID index
|
||||
|
||||
or has expressed by any other means his willingness to license under the EUPL.
|
||||
The UID index is an application of the Bayou storage module
|
||||
used to assign UID numbers to e-mails.
|
||||
See document we sent to NGI for properties on UIDVALIDITY.
|
||||
|
||||
## Cryptography; key management
|
||||
|
||||
Keys that are used:
|
||||
|
||||
- master secret key (for indexes)
|
||||
- curve25519 public/private key pair (for incoming mail)
|
||||
|
||||
Keys that are stored in K2V under PK `keys`:
|
||||
|
||||
- `public`: the public curve25519 key (plain text)
|
||||
- `salt`: the 32-byte salt `S` used to calculate digests that index keys below
|
||||
- if a password is used, `password:<truncated(128bit) argon2 digest of password using salt S>`:
|
||||
- a 32-byte salt `Skey`
|
||||
- followed a secret box
|
||||
- that is encrypted with a strong argon2 digest of the password (using the salt `Skey`) and a user secret (see below)
|
||||
- that contains the master secret key and the curve25519 private key
|
||||
|
||||
User secret: an additionnal secret that is added to the password when deriving the encryption key for the secret box.
|
||||
This additionnal secret should not be stored in K2V/S3, so that just knowing a user's password isn't enough to be able
|
||||
to decrypt their mailbox (supposing the attacker has a dump of their K2V/S3 bucket).
|
||||
This user secret should typically be stored in the LDAP database or just in the configuration file when using
|
||||
the static login provider.
|
||||
|
||||
Operations:
|
||||
|
||||
- **Initialize**(`user_secret`, `password`):
|
||||
- if `"salt"` or `"public"` already exist, BAIL
|
||||
- generate salt `S` (32 random bytes)
|
||||
- generate `public`, `private` (curve25519 keypair)
|
||||
- generate `master` (secretbox secret key)
|
||||
- calculate `digest = argon2_S(password)`
|
||||
- generate salt `Skey` (32 random bytes)
|
||||
- calculate `key = argon2_Skey(user_secret + password)`
|
||||
- serialize `box_contents = (private, master)`
|
||||
- seal box `blob = seal_key(box_contents)`
|
||||
- write `S` at `"salt"`
|
||||
- write `concat(Skey, blob)` at `"password:{hex(digest[..16])}"`
|
||||
- write `public` at `"public"`
|
||||
|
||||
- **InitializeWithoutPassword**(`private`, `master`):
|
||||
- if `"salt"` or `"public"` already exist, BAIL
|
||||
- generate salt `S` (32 random bytes)
|
||||
- write `S` at `"salt"`
|
||||
- calculate `public` the public key associated with `private`
|
||||
- write `public` at `"public"`
|
||||
|
||||
- **Open**(`user_secret`, `password`):
|
||||
- load `S = read("salt")`
|
||||
- calculate `digest = argon2_S(password)`
|
||||
- load `blob = read("password:{hex(digest[..16])}")
|
||||
- set `Skey = blob[..32]`
|
||||
- calculate `key = argon2_Skey(user_secret + password)`
|
||||
- open secret box `box_contents = open_key(blob[32..])`
|
||||
- retrieve `master` and `private` from `box_contents`
|
||||
- retrieve `public = read("public")`
|
||||
|
||||
- **OpenWithoutPassword**(`private`, `master`):
|
||||
- load `public = read("public")`
|
||||
- check that `public` is the correct public key associated with `private`
|
||||
|
||||
- **AddPassword**(`user_secret`, `existing_password`, `new_password`):
|
||||
- load `S = read("salt")`
|
||||
- calculate `digest = argon2_S(existing_password)`
|
||||
- load `blob = read("existing_password:{hex(digest[..16])}")
|
||||
- set `Skey = blob[..32]`
|
||||
- calculate `key = argon2_Skey(user_secret + existing_password)`
|
||||
- open secret box `box_contents = open_key(blob[32..])`
|
||||
- retrieve `master` and `private` from `box_contents`
|
||||
|
||||
- calculate `digest_new = argon2_S(new_password)`
|
||||
- generate salt `Skeynew` (32 random bytes)
|
||||
- calculate `key_new = argon2_Skeynew(user_secret + new_password)`
|
||||
- serialize `box_contents_new = (private, master)`
|
||||
- seal box `blob_new = seal_key_new(box_contents_new)`
|
||||
- write `concat(Skeynew, blob_new)` at `"new_password:{hex(digest_new[..16])}"`
|
||||
|
||||
- **RemovePassword**(`password`):
|
||||
- load `S = read("salt")`
|
||||
- calculate `digest = argon2_S(existing_password)`
|
||||
- check that `"password:{hex(digest[..16])}"` exists
|
||||
- check that other passwords exist ?? (or not)
|
||||
- delete `"password:{hex(digest[..16])}"`
|
||||
|
|
|
|||
1
doc/.gitignore
vendored
1
doc/.gitignore
vendored
|
|
@ -1 +0,0 @@
|
|||
book
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
[book]
|
||||
authors = ["Quentin Dufour"]
|
||||
language = "en"
|
||||
multilingual = false
|
||||
src = "src"
|
||||
title = "Aerogramme - Encrypted e-mail storage over Garage"
|
||||
|
||||
[output.html]
|
||||
mathjax-support = true
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
# Summary
|
||||
|
||||
[Introduction](./index.md)
|
||||
|
||||
# Quick start
|
||||
|
||||
- [Installation](./installation.md)
|
||||
- [Setup](./setup.md)
|
||||
- [Validation](./validate.md)
|
||||
|
||||
# Cookbook
|
||||
|
||||
- [Not ready for production]()
|
||||
|
||||
# Reference
|
||||
|
||||
- [Configuration file](./config.md)
|
||||
- [RFC coverage](./rfc.md)
|
||||
|
||||
# Design
|
||||
|
||||
- [Overview](./overview.md)
|
||||
- [Mailboxes](./mailbox.md)
|
||||
- [Mutation Log](./log.md)
|
||||
- [IMAP UID proof](./imap_uid.md)
|
||||
|
||||
# Internals
|
||||
|
||||
- [Persisted data structures](./data_format.md)
|
||||
- [Cryptography & key management](./crypt-key.md)
|
||||
|
||||
# Development
|
||||
|
||||
- [Notes](./notes.md)
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 26 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 27 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 73 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 8.9 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 18 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 550 KiB |
|
|
@ -1,126 +0,0 @@
|
|||
# Configuration file
|
||||
|
||||
A configuration file that illustrate all the possible options,
|
||||
in practise, many fields are omitted:
|
||||
|
||||
```toml
|
||||
s3_endpoint = "s3.garage.tld"
|
||||
k2v_endpoint = "k2v.garage.tld"
|
||||
aws_region = "garage"
|
||||
|
||||
[lmtp]
|
||||
bind_addr = "[::1]:2525"
|
||||
hostname = "aerogramme.tld"
|
||||
|
||||
[imap]
|
||||
bind_addr = "[::1]:993"
|
||||
|
||||
[login_static]
|
||||
default_bucket = "aerogramme"
|
||||
|
||||
[login_static.user.alan]
|
||||
email_addresses = [
|
||||
"alan@smith.me"
|
||||
"aln@example.com"
|
||||
]
|
||||
password = "$argon2id$v=19$m=4096,t=3,p=1$..."
|
||||
|
||||
aws_access_key_id = "GK..."
|
||||
aws_secret_access_key = "c0ffee"
|
||||
bucket = "aerogramme-alan"
|
||||
|
||||
user_secret = "s3cr3t"
|
||||
alternate_user_secrets = [ "s3cr3t2" "s3cr3t3" ]
|
||||
|
||||
master_key = "..."
|
||||
secret_key = "..."
|
||||
|
||||
[login_ldap]
|
||||
ldap_server = "ldap.example.com"
|
||||
|
||||
pre_bind_on_login = true
|
||||
bind_dn = "cn=admin,dc=example,dc=com"
|
||||
bind_password = "s3cr3t"
|
||||
|
||||
search_base = "ou=users,dc=example,dc=com"
|
||||
username_attr = "cn"
|
||||
mail_attr = "mail"
|
||||
|
||||
aws_access_key_id_attr = "garage_s3_access_key"
|
||||
aws_secret_access_key_attr = "garage_s3_secret_key"
|
||||
user_secret_attr = "secret"
|
||||
alternate_user_secrets_attr = "secret_alt"
|
||||
|
||||
# bucket = "aerogramme"
|
||||
bucket_attr = "bucket"
|
||||
|
||||
```
|
||||
|
||||
## Global configuration options
|
||||
|
||||
### `s3_endpoint`
|
||||
|
||||
### `k2v_endpoint`
|
||||
|
||||
### `aws_region`
|
||||
|
||||
## LMTP configuration options
|
||||
|
||||
### `lmtp.bind_addr`
|
||||
|
||||
### `lmtp.hostname`
|
||||
|
||||
## IMAP configuration options
|
||||
|
||||
### `imap.bind_addr`
|
||||
|
||||
## Static login configuration options
|
||||
|
||||
### `login_static.default_bucket`
|
||||
|
||||
### `login_static.user.<name>.email_addresses`
|
||||
|
||||
### `login_static.user.<name>.password`
|
||||
|
||||
### `login_static.user.<name>.aws_access_key_id`
|
||||
|
||||
### `login_static.user.<name>.aws_secret_access_key`
|
||||
|
||||
### `login_static.user.<name>.bucket`
|
||||
|
||||
### `login_static.user.<name>.user_secret`
|
||||
|
||||
### `login_static.user.<name>.master_key`
|
||||
|
||||
### `login_static.user.<name>.secret_key`
|
||||
|
||||
## LDAP login configuration options
|
||||
|
||||
### `login_ldap.ldap_server`
|
||||
|
||||
### `login_ldap.pre_bind_on`
|
||||
|
||||
### `login_ldap.bind_dn`
|
||||
|
||||
### `login_ldap.bind_password`
|
||||
|
||||
### `login_ldap.search_base`
|
||||
|
||||
### `login_ldap.username_attr`
|
||||
|
||||
### `login_ldap.mail_attr`
|
||||
|
||||
### `login_ldap.aws_access_key_id_attr`
|
||||
|
||||
### `login_ldap.aws_secret_access_key_attr`
|
||||
|
||||
### `login_ldap.user_secret_attr`
|
||||
|
||||
### `login_ldap.alternate_user_secrets_attr`
|
||||
|
||||
### `login_ldap.bucket`
|
||||
|
||||
### `login_ldap.bucket_attr`
|
||||
|
||||
|
||||
|
||||
|
|
@ -1,82 +0,0 @@
|
|||
# Cryptography & key management
|
||||
|
||||
Keys that are used:
|
||||
|
||||
- master secret key (for indexes)
|
||||
- curve25519 public/private key pair (for incoming mail)
|
||||
|
||||
Keys that are stored in K2V under PK `keys`:
|
||||
|
||||
- `public`: the public curve25519 key (plain text)
|
||||
- `salt`: the 32-byte salt `S` used to calculate digests that index keys below
|
||||
- if a password is used, `password:<truncated(128bit) argon2 digest of password using salt S>`:
|
||||
- a 32-byte salt `Skey`
|
||||
- followed a secret box
|
||||
- that is encrypted with a strong argon2 digest of the password (using the salt `Skey`) and a user secret (see below)
|
||||
- that contains the master secret key and the curve25519 private key
|
||||
|
||||
User secret: an additionnal secret that is added to the password when deriving the encryption key for the secret box.
|
||||
This additionnal secret should not be stored in K2V/S3, so that just knowing a user's password isn't enough to be able
|
||||
to decrypt their mailbox (supposing the attacker has a dump of their K2V/S3 bucket).
|
||||
This user secret should typically be stored in the LDAP database or just in the configuration file when using
|
||||
the static login provider.
|
||||
|
||||
Operations:
|
||||
|
||||
- **Initialize**(`user_secret`, `password`):
|
||||
- if `"salt"` or `"public"` already exist, BAIL
|
||||
- generate salt `S` (32 random bytes)
|
||||
- generate `public`, `private` (curve25519 keypair)
|
||||
- generate `master` (secretbox secret key)
|
||||
- calculate `digest = argon2_S(password)`
|
||||
- generate salt `Skey` (32 random bytes)
|
||||
- calculate `key = argon2_Skey(user_secret + password)`
|
||||
- serialize `box_contents = (private, master)`
|
||||
- seal box `blob = seal_key(box_contents)`
|
||||
- write `S` at `"salt"`
|
||||
- write `concat(Skey, blob)` at `"password:{hex(digest[..16])}"`
|
||||
- write `public` at `"public"`
|
||||
|
||||
- **InitializeWithoutPassword**(`private`, `master`):
|
||||
- if `"salt"` or `"public"` already exist, BAIL
|
||||
- generate salt `S` (32 random bytes)
|
||||
- write `S` at `"salt"`
|
||||
- calculate `public` the public key associated with `private`
|
||||
- write `public` at `"public"`
|
||||
|
||||
- **Open**(`user_secret`, `password`):
|
||||
- load `S = read("salt")`
|
||||
- calculate `digest = argon2_S(password)`
|
||||
- load `blob = read("password:{hex(digest[..16])}")
|
||||
- set `Skey = blob[..32]`
|
||||
- calculate `key = argon2_Skey(user_secret + password)`
|
||||
- open secret box `box_contents = open_key(blob[32..])`
|
||||
- retrieve `master` and `private` from `box_contents`
|
||||
- retrieve `public = read("public")`
|
||||
|
||||
- **OpenWithoutPassword**(`private`, `master`):
|
||||
- load `public = read("public")`
|
||||
- check that `public` is the correct public key associated with `private`
|
||||
|
||||
- **AddPassword**(`user_secret`, `existing_password`, `new_password`):
|
||||
- load `S = read("salt")`
|
||||
- calculate `digest = argon2_S(existing_password)`
|
||||
- load `blob = read("existing_password:{hex(digest[..16])}")
|
||||
- set `Skey = blob[..32]`
|
||||
- calculate `key = argon2_Skey(user_secret + existing_password)`
|
||||
- open secret box `box_contents = open_key(blob[32..])`
|
||||
- retrieve `master` and `private` from `box_contents`
|
||||
|
||||
- calculate `digest_new = argon2_S(new_password)`
|
||||
- generate salt `Skeynew` (32 random bytes)
|
||||
- calculate `key_new = argon2_Skeynew(user_secret + new_password)`
|
||||
- serialize `box_contents_new = (private, master)`
|
||||
- seal box `blob_new = seal_key_new(box_contents_new)`
|
||||
- write `concat(Skeynew, blob_new)` at `"new_password:{hex(digest_new[..16])}"`
|
||||
|
||||
- **RemovePassword**(`password`):
|
||||
- load `S = read("salt")`
|
||||
- calculate `digest = argon2_S(existing_password)`
|
||||
- check that `"password:{hex(digest[..16])}"` exists
|
||||
- check that other passwords exist ?? (or not)
|
||||
- delete `"password:{hex(digest[..16])}"`
|
||||
|
|
@ -1,50 +0,0 @@
|
|||
# Data format
|
||||
|
||||
## Bay(ou)
|
||||
|
||||
Checkpoints are stored in S3 at `<path>/checkpoint/<timestamp>`. Example:
|
||||
|
||||
```
|
||||
348 TestMailbox/checkpoint/00000180d77400dc126b16aac546b769
|
||||
369 TestMailbox/checkpoint/00000180d776e509b68fdc5c376d0abc
|
||||
357 TestMailbox/checkpoint/00000180d77a7fe68f4f76e3b45aa751
|
||||
```
|
||||
|
||||
Operations are stored in K2V at PK `<path>`, SK `<timestamp>`. Example:
|
||||
|
||||
```
|
||||
TestMailbox 00000180d77400dc126b16aac546b769 RcIsESv7WrjMuHwyI/dvCnkIfy6op5Tiylf0WSnn94aMS2uagl7YeMBwdv09TiSXBpu5nJ5e/9QFSfuEI/NqKrdQkX54MOsnaIGhRb0oqUG3KNaar3BiVSvYvXuzYhk4ii+TUS2Eyd6fCCaNVNM5
|
||||
TestMailbox 00000180d775f27f5542a13fc21c665e RrTSOup/zO1Ei+QrjBcDLt4vvFSY+WJPBodwY64wy2ftW+Oh3VSArvlO4SAEPmdsx1gt0HPBZYR/OkVWsZpmix1ZLFUmvdib+rjNkorHQW1p+oLVK8tolGrqk4SRwl88cqu466T4vBEpDu7tRbH0
|
||||
TestMailbox 00000180d775f292b3c8da00718389b4 VAwd8SRycIwsipZW5AcSG+EIYZVWn/Uj/TADbWhb4x5LVMceiRBHWVquY08RgT/lJKdhIcUqBA15bVG3klIg8tLsWJVG784NbsZwdGRczWmngcA=
|
||||
TestMailbox 00000180d775f29d24842cf375d679e0 /FbXtEwm/bijtvOdqM1XFvKUalQFAOPHp+vF9jZThZn/viY5a6W1PyHeI8kTusF6EsVPAwPHpQyjIv/ghskC0f+zUEsSUhDwQANdwLNqDLAvTA==
|
||||
TestMailbox 00000180d7768ab1dc01ff504e887c62 W/fF0WitpxJ05yHeOv96BlpGymT1kVOjkIW00t9e6UE7mxkvNflu9cZSCd8PDJd2ymC0sC9bLVFAXKmNZsmCFEEHMQSyrX61qTYo4KFCZMp5zm6fXubaYuurrzjXzfUP/R7kBvICFZlF0daf0SwX
|
||||
TestMailbox 00000180d7768aba629c7ad6adf25228 IPzYGNsSepCX2AEnee/1Eas9a3c5esPSmrNkvaj4XcFb6Ft2KC8N6ubUR3wB+K0oYCTQym6nhHG5dlAxf6NRu7Rk8YtBTBmSqtGqd6kMZ3bU5b8=
|
||||
TestMailbox 00000180d7768ac1870cda61784114d4 aaLiaWxfx1mxh6aoKE3xUUfZWhivZ/K7ixabflFDW7FO/qbpvCaa+Y6w4lQemTy6m+leAhXGN+Dbyv2qP20yJ9O4oJF5d3Lz5Iv5uF18OxhVZzw=
|
||||
TestMailbox 00000180d776e4fb294ccdab2612b406 EtUPrLgEeOyab2QRnSie4I3Me9dDh10UdwWnUKdGa/8ezMJDtiy7XlW+tUfJdqtu6Vj7nduT0emDOXbBZsNwlcmzgYNwuNu3I9AfhZTFWtwLgB+wnAgB/jim82DDrJfLia8kB2eA2ao5jfJ3uMSZ
|
||||
TestMailbox 00000180d776e501528546d340490291 Lz4Z9wCTk1lZ86lL01urhAan4oHcr1NBqdRe+CDpA51D9IncA5+Fhc8I6knUIh2qQ5/woWgISLAVwzSS+0+TxrYoqxf5FumIQtUJfwDER5La3n0=
|
||||
TestMailbox 00000180d776e509b68fdc5c376d0abc RUGE2xB3fFX/wRH/p2fHIUa+rMaXSRd7fY9zglw0pRfVPqJfpniOjAe4GHIwGlwbwjtFOwS5a+Q7yr0Wez6QwD+ohhqRFKpbjcFcN7VfMyVAf+k=
|
||||
TestMailbox 00000180d7784b987a8ad8106dc400c9 K+0LVEtBbTnWNS67jy9DtTvQyd5arovduvu490tLOE2TzVhuVoF4pfvTMTN12bH3KwEAHeDfuwKkKJFqldOywouTYPzEjZFkJzyagHrkl6dfnE5CqmlDv+Vc5TOQRskxjW+wQiZdjU8wGiBiBGYh
|
||||
TestMailbox 00000180d7784bede69ac3cff2c6b724 XMFY3+b1r1//uolVz80JSI3g/84XCk3Tm7/S0BFv+Qe/Xv3/poLrOvAKEe+GzD2s22j8p/T2RXR/JSZckzgjEZeO0wbPDXVQd94di2Pff7jxAH8=
|
||||
TestMailbox 00000180d7784bffe2595abe7ed81858 QQZhF+7wSHfikoAp93a+UY/XDIX7TVnnVYOtmQ2XHnDKA2F6snRJCPbYBO4IRHCRfVrjDGi32c41it2C3Mu5PBepabxapsW1rfIV3rlX2lkKHtI=
|
||||
TestMailbox 00000180d77a7fb3f01dbb147c20cf7f IHOlOa1JI11RUKVvQUq3HQPxiRr4UCeE+pHmL8DtNMkOh62V4spuP0VvvQTJCQcPQ1EQR/QcxZ3s7uHLkrZAHF30BkpUkGqsLBWpnyug/puhdiixWsMyLLb6G90zFjiComUwptnDc/CCXtGEHdSW
|
||||
TestMailbox 00000180d77a7fbb54b100f521ceb347 Ze4KyyTCgrYbZlXlJSY5hNob8sMXvBAmwIx2cADbX5P0M1IHXwXfloEzvvd6WYOtatFC2GnDSrmQ6RdCfeZ3WV9TZilqa0Fv0XEg48sVyVCcguw=
|
||||
TestMailbox 00000180d77a7fe68f4f76e3b45aa751 cJJVvvRzTVNKUaIHPCCDY2uY7/HlmkxGgo3ozWBlBSRDeBqU65zgZD3QIPCxa6xaqB/Gc0bQ9BGzfU0cvVmO5jgNeeDnbqqs3oeA2jml/Qv2YO9upApfNQtDT1GiwJ8vrgaIow==
|
||||
TestMailbox 00000180d8e513d3ea58c679a13178ac Ce5su2YOxNmTzk2dK8SX8V/Uue5uAC7oklEjhesY9wCMqGphhOkdWjzCqq0xOzcb/ZzzZ58t+mTksNSYIU4kddHIHBFPgqIwKthVk2mlUdqYiN/Y2vEGqv+YmtKY+GST/7Ee87ZHpU/5sv0GoXxT
|
||||
TestMailbox 00000180d8e5145a23f8faee86283900 sp3D8xFZcM9icNlDJXIUDJb3mo6VGD9f1aDHD+4RbPdx6mTYF+qNTsPHKCxHHxT/9NfNe8XPg2+8xYRtm7SXfgERZBDB8ye+Xt3fM1k+wbL6RsaJmDHVECeXeL5KHuITzpI22A==
|
||||
TestMailbox 00000180d8e51465c38f0585f9bb760e FF0VId2O/bBNzYD5ABWReMs5hHoHwynOoJRKj9vyaUMZ3JykInFmvvRgtCbJBDjTQPwPU8apphKQfwuicO76H7GtZqH009Cbv5l8ZTRJKrmzOQmtjzBQc2eGEUMPfbml5t0GCg==
|
||||
```
|
||||
|
||||
The timestamp of a checkpoint corresponds to the timestamp of the first operation NOT included in the checkpoint.
|
||||
In other words, to reconstruct the final state:
|
||||
|
||||
- find timestamp `<ts>` of last checkpoint
|
||||
- load checkpoint `<ts>`
|
||||
- load and apply all operations starting from `<ts>`, included
|
||||
|
||||
## UID index
|
||||
|
||||
The UID index is an application of the Bayou storage module
|
||||
used to assign UID numbers to e-mails.
|
||||
See document we sent to NGI for properties on UIDVALIDITY.
|
||||
|
||||
|
||||
|
|
@ -1,203 +0,0 @@
|
|||
# IMAP UID proof
|
||||
|
||||
**Notations**
|
||||
|
||||
- $h$: the hash of a message, $\mathbb{H}$ is the set of hashes
|
||||
- $i$: the UID of a message $(i \in \mathbb{N})$
|
||||
- $f$: a flag attributed to a message (it's a string), we write
|
||||
$\mathbb{F}$ the set of possible flags
|
||||
- if $M$ is a map (aka a dictionnary), if $x$ has no assigned value in
|
||||
$M$ we write $M [x] = \bot$ or equivalently $x \not\in M$. If $x$ has a value
|
||||
in the map we write $x \in M$ and $M [x] \neq \bot$
|
||||
|
||||
**State**
|
||||
|
||||
- A map $I$ such that $I [h]$ is the UID of the message whose hash is
|
||||
$h$ is the mailbox, or $\bot$ if there is no such message
|
||||
|
||||
- A map $F$ such that $F [h]$ is the set of flags attributed to the
|
||||
message whose hash is $h$
|
||||
|
||||
- $v$: the UIDVALIDITY value
|
||||
|
||||
- $n$: the UIDNEXT value
|
||||
|
||||
- $s$: an internal sequence number that is mostly equal to UIDNEXT but
|
||||
also grows when mails are deleted
|
||||
|
||||
**Operations**
|
||||
|
||||
- MAIL\_ADD$(h, i)$: the value of $i$ that is put in this operation is
|
||||
the value of $s$ in the state resulting of all already known operations,
|
||||
i.e. $s (O_{gen})$ in the notation below where $O_{gen}$ is
|
||||
the set of all operations known at the time when the MAIL\_ADD is generated.
|
||||
Moreover, such an operation can only be generated if $I (O_{gen}) [h]
|
||||
= \bot$, i.e. for a mail $h$ that is not already in the state at
|
||||
$O_{gen}$.
|
||||
|
||||
- MAIL\_DEL$(h)$
|
||||
|
||||
- FLAG\_ADD$(h, f)$
|
||||
|
||||
- FLAG\_DEL$(h, f)$
|
||||
|
||||
**Algorithms**
|
||||
|
||||
|
||||
**apply** MAIL\_ADD$(h, i)$:
|
||||
*if* $i < s$:
|
||||
$v \leftarrow v + s - i$
|
||||
*if* $F [h] = \bot$:
|
||||
$F [h] \leftarrow F_{initial}$
|
||||
$I [h] \leftarrow s$
|
||||
$s \leftarrow s + 1$
|
||||
$n \leftarrow s$
|
||||
|
||||
**apply** MAIL\_DEL$(h)$:
|
||||
$I [h] \leftarrow \bot$
|
||||
$F [h] \leftarrow \bot$
|
||||
$s \leftarrow s + 1$
|
||||
|
||||
**apply** FLAG\_ADD$(h, f)$:
|
||||
*if* $h \in F$:
|
||||
$F [h] \leftarrow F [h] \cup \{ f \}$
|
||||
|
||||
**apply** FLAG\_DEL$(h, f)$:
|
||||
*if* $h \in F$:
|
||||
$F [h] \leftarrow F [h] \backslash \{ f \}$
|
||||
|
||||
|
||||
**More notations**
|
||||
|
||||
- $o$ is an operation such as MAIL\_ADD, MAIL\_DEL, etc. $O$ is a set of
|
||||
operations. Operations embed a timestamp, so a set of operations $O$ can be
|
||||
written as $O = [o_1, o_2, \ldots, o_n]$ by ordering them by timestamp.
|
||||
|
||||
- if $o \in O$, we write $O_{\leqslant o}$, $O_{< o}$, $O_{\geqslant
|
||||
o}$, $O_{> o}$ the set of items of $O$ that are respectively earlier or
|
||||
equal, strictly earlier, later or equal, or strictly later than $o$. In
|
||||
other words, if we write $O = [o_1, \ldots, o_n]$, where $o$ is a certain
|
||||
$o_i$ in this sequence, then:
|
||||
$$
|
||||
\begin{aligned}
|
||||
O_{\leqslant o} &= \{ o_1, \ldots, o_i \}\\
|
||||
O_{< o} &= \{ o_1, \ldots, o_{i - 1} \}\\
|
||||
O_{\geqslant o} &= \{ o_i, \ldots, o_n \}\\
|
||||
O_{> o} &= \{ o_{i + 1}, \ldots, o_n \}
|
||||
\end{aligned}
|
||||
$$
|
||||
|
||||
- If $O$ is a set of operations, we write $I (O)$, $F (O)$, $n (O), s
|
||||
(O)$, and $v (O)$ the values of $I, F, n, s$ and $v$ in the state that
|
||||
results of applying all of the operations in $O$ in their sorted order. (we
|
||||
thus write $I (O) [h]$ the value of $I [h]$ in this state)
|
||||
|
||||
**Hypothesis:**
|
||||
An operation $o$ can only be in a set $O$ if it was
|
||||
generated after applying operations of a set $O_{gen}$ such that
|
||||
$O_{gen} \subset O$ (because causality is respected in how we deliver
|
||||
operations). Sets of operations that do not respect this property are excluded
|
||||
from all of the properties, lemmas and proofs below.
|
||||
|
||||
**Simplification:** We will now exclude FLAG\_ADD and FLAG\_DEL
|
||||
operations, as they do not manipulate $n$, $s$ and $v$, and adding them should
|
||||
have no impact on the properties below.
|
||||
|
||||
**Small lemma:** If there are no FLAG\_ADD and FLAG\_DEL operations,
|
||||
then $s (O) = | O |$. This is easy to see because the possible operations are
|
||||
only MAIL\_ADD and MAIL\_DEL, and both increment the value of $s$ by 1.
|
||||
|
||||
**Defnition:** If $o$ is a MAIL\_ADD$(h, i)$ operation, and $O$ is a
|
||||
set of operations such that $o \in O$, then we define the following value:
|
||||
$$
|
||||
C (o, O) = s (O_{< o}) - i
|
||||
$$
|
||||
We say that $C (o, O)$ is the *number of conflicts of $o$ in $O$*: it
|
||||
corresponds to the number of operations that were added before $o$ in $O$ that
|
||||
were not in $O_{gen}$.
|
||||
|
||||
**Property:**
|
||||
|
||||
We have that:
|
||||
|
||||
$$
|
||||
v (O) = \sum_{o \in O} C (o, O)
|
||||
$$
|
||||
|
||||
Or in English: $v (O)$ is the sum of the number of conflicts of all of the
|
||||
MAIL\_ADD operations in $O$. This is easy to see because indeed $v$ is
|
||||
incremented by $C (o, O)$ for each operation $o \in O$ that is applied.
|
||||
|
||||
|
||||
**Property:**
|
||||
If $O$ and $O'$ are two sets of operations, and $O \subseteq O'$, then:
|
||||
|
||||
$$
|
||||
\begin{aligned}
|
||||
\forall o \in O, \qquad C (o, O) \leqslant C (o, O')
|
||||
\end{aligned}
|
||||
$$
|
||||
|
||||
This is easy to see because $O_{< o} \subseteq O'_{< o}$ and $C (o, O') - C
|
||||
(o, O) = s (O'_{< o}) - s (O_{< o}) = | O'_{< o} | - | O_{< o} | \geqslant
|
||||
0$
|
||||
|
||||
**Theorem:**
|
||||
|
||||
If $O$ and $O'$ are two sets of operations:
|
||||
|
||||
$$
|
||||
\begin{aligned}
|
||||
O \subseteq O' & \Rightarrow & v (O) \leqslant v (O')
|
||||
\end{aligned}
|
||||
$$
|
||||
|
||||
**Proof:**
|
||||
|
||||
$$
|
||||
\begin{aligned}
|
||||
v (O') &= \sum_{o \in O'} C (o, O')\\
|
||||
& \geqslant \sum_{o \in O} C (o, O') \qquad \text{(because $O \subseteq
|
||||
O'$)}\\
|
||||
& \geqslant \sum_{o \in O} C (o, O) \qquad \text{(because $\forall o \in
|
||||
O, C (o, O) \leqslant C (o, O')$)}\\
|
||||
& \geqslant v (O)
|
||||
\end{aligned}
|
||||
$$
|
||||
|
||||
**Theorem:**
|
||||
|
||||
If $O$ and $O'$ are two sets of operations, such that $O \subset O'$,
|
||||
|
||||
and if there are two different mails $h$ and $h'$ $(h \neq h')$ such that $I
|
||||
(O) [h] = I (O') [h']$
|
||||
|
||||
then:
|
||||
$$v (O) < v (O')$$
|
||||
|
||||
**Proof:**
|
||||
|
||||
We already know that $v (O) \leqslant v (O')$ because of the previous theorem.
|
||||
We will now look at the sum:
|
||||
$$
|
||||
v (O') = \sum_{o \in O'} C (o, O')
|
||||
$$
|
||||
and show that there is at least one term in this sum that is strictly larger
|
||||
than the corresponding term in the other sum:
|
||||
$$
|
||||
v (O) = \sum_{o \in O} C (o, O)
|
||||
$$
|
||||
Let $o$ be the last MAIL\_ADD$(h, \_)$ operation in $O$, i.e. the operation
|
||||
that gives its definitive UID to mail $h$ in $O$, and similarly $o'$ be the
|
||||
last MAIL\_ADD($h', \_$) operation in $O'$.
|
||||
|
||||
Let us write $I = I (O) [h] = I (O') [h']$
|
||||
|
||||
$o$ is the operation at position $I$ in $O$, and $o'$ is the operation at
|
||||
position $I$ in $O'$. But $o \neq o'$, so if $o$ is not the operation at
|
||||
position $I$ in $O'$ then it has to be at a later position $I' > I$ in $O'$,
|
||||
because no operations are removed between $O$ and $O'$, the only possibility
|
||||
is that some other operations (including $o'$) are added before $o$. Therefore
|
||||
we have that $C (o, O') > C (o, O)$, i.e. at least one term in the sum above
|
||||
is strictly larger in the first sum than in the second one. Since all other
|
||||
terms are greater or equal, we have $v (O') > v (O)$.
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
# Introduction
|
||||
|
||||
<p align="center" style="text-align:center;">
|
||||
<img alt="A scan of an Aerogramme dating from 1955" src="./aerogramme.jpg" style="margin:auto; max-width:300px"/>
|
||||
<br>
|
||||
[ <strong><a href="https://aerogramme.deuxfleurs.fr/">Documentation</a></strong>
|
||||
| <a href="https://git.deuxfleurs.fr/Deuxfleurs/aerogramme">Git repository</a>
|
||||
]
|
||||
<br>
|
||||
<em>stability status: technical preview (do not use in production)</em>
|
||||
</p>
|
||||
|
||||
Aerogramme is an open-source **IMAP server** targeted at **distributed** infrastructures and written in **Rust**.
|
||||
It is designed to be resilient, easy to operate and private by design.
|
||||
|
||||
**Resilient** - Aerogramme is built on top of Garage, a (geographically) distributed object storage software. Aerogramme thus inherits Garage resiliency: its mailboxes are spread on multiple distant regions, regions can go offline while keeping mailboxes available, storage nodes can be added or removed on the fly, etc.
|
||||
|
||||
**Easy to operate** - Aerogramme mutualizes the burden of data management by storing all its data in an object store and nothing on the local filesystem or any relational database. It can be seen as a proxy between the IMAP protocol and Garage protocols (S3 and K2V). It can thus be freely moved between machines. Multiple instances can also be run in parallel.
|
||||
|
||||
**Private by design** - As emails are very sensitive, Aerogramme encrypts users' mailboxes with their passwords. Data is decrypted in RAM upon user login: the Garage storage layer handles only encrypted blobs. It is even possible to run locally Aerogramme while connecting it to a remote, third-party, untrusted Garage provider; in this case clear text emails never leak outside of your computer.
|
||||
|
||||
Our main use case is to provide a modern email stack for autonomously hosted communities such as [Deuxfleurs](https://deuxfleurs.fr). More generally, we want to set new standards in term of email ethic by lowering the bar to become an email provider while making it harder to spy users' emails.
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
# Installation
|
||||
|
||||
Install a Rust nightly toolchain: [go to Rustup](https://rustup.rs/).
|
||||
|
||||
Install and deploy a Garage cluster: [go to Garage documentation](https://garagehq.deuxfleurs.fr/documentation/quick-start/). Make sure that you download a binary that supports K2V. Currently, you will find them in the "Extra build" section of the Download page.
|
||||
|
||||
Clone Aerogramme's repository:
|
||||
|
||||
```bash
|
||||
git clone https://git.deuxfleurs.fr/Deuxfleurs/aerogramme/
|
||||
```
|
||||
|
||||
Compile Aerogramme:
|
||||
|
||||
```bash
|
||||
cargo build
|
||||
```
|
||||
|
||||
Check that your compiled binary works:
|
||||
|
||||
```bash
|
||||
cargo run
|
||||
```
|
||||
|
||||
You are now ready to [setup Aerogramme!](./setup.md)
|
||||
149
doc/src/log.md
149
doc/src/log.md
|
|
@ -1,149 +0,0 @@
|
|||
# Mutation Log
|
||||
|
||||
|
||||
Back to our data structure, we note that one major challenge with this project is to *correctly* handle mutable data.
|
||||
With our current design, multiple processes can interact with the same mutable data without coordination, and we need a way to detect and solve conflicts.
|
||||
Directly storing the result in a single k2v key would not work as we have no transaction or lock mechanism, and our state would be always corrupted.
|
||||
Instead, we choose to record an ordered log of operations, ie. transitions, that each client can use locally to rebuild the state, each transition has its own immutable identifier.
|
||||
This technique is sometimes referred to as event sourcing.
|
||||
|
||||
With this system, we can't have conflict anymore at Garage level, but conflicts at the IMAP level can still occur, like 2 processes assigning the same identifier to different emails.
|
||||
We thus need a logic to handle these conflicts that is flexible enough to accommodate the application's specific logic.
|
||||
|
||||
Our solution is inspired by the work conducted by Terry et al. on [Bayou](https://dl.acm.org/doi/10.1145/224056.224070).
|
||||
Clients fetch regularly the log from Garage, each entry is ordered by a timestamp and a unique identifier.
|
||||
One of the 2 conflicting clients will be in the state where it has executed a log entry in the wrong order according to the specified ordering.
|
||||
This client will need to roll back its changes to reapply the log in the same order as the others, and on conflicts, the same logic will be applied by all the clients to get, in the end, the same state.
|
||||
|
||||
**Command definitions**
|
||||
|
||||
The log is made of a sequence of ordered commands that can be run to get a deterministic state in the end.
|
||||
We define the following commands:
|
||||
|
||||
`FLAG_ADD <email_uuid> <flag>` - Add a flag to the target email
|
||||
`FLAG_DEL <email_uuid> <flag>` - Remove a flag from a target email
|
||||
`MAIL_DEL <email_uuid>` - Remove an email
|
||||
`MAIL_ADD <email_uuid> <uid>` - Register an email in the mailbox with the given identifier
|
||||
`REMOTE <s3 url>` - Command is not directly stored here, instead it must be fetched from S3, see batching to understand why.
|
||||
|
||||
*Note: FLAG commands could be enhanced with a MODSEQ field similar to the uid field for the emails, in order to implement IMAP RFC4551. Adding this field would force us to handle conflicts on flags
|
||||
the same way as on emails, as MODSEQ must be monotonically incremented but is reset by a uid-validity change. This is out of the scope of this document.*
|
||||
|
||||
**A note on UUID**
|
||||
|
||||
When adding an email to the system, we associate it with a *universally unique identifier* or *UUID.*
|
||||
We can then reference this email in the rest of the system without fearing a conflict or a race condition are we are confident that this UUID is unique.
|
||||
|
||||
We could have used the email hash instead, but we identified some benefits in using UUID.
|
||||
First, sometimes a mail must be duplicated, because the user received it from 2 different sources, so it is more correct to have 2 entries in the system.
|
||||
Additionally, UUIDs are smaller and better compressible than a hash, which will lead to better performances.
|
||||
|
||||
**Batching commands**
|
||||
|
||||
Commands that are executed at the same time can be batched together.
|
||||
Let's imagine a user is deleting its trash containing thousands of emails.
|
||||
Instead of writing thousands of log lines, we can append them in a single entry.
|
||||
If this entry becomes big (eg. > 100 commands), we can store it to S3 with the `REMOTE` command.
|
||||
Batching is important as we want to keep the number of log entries small to be able to fetch them regularly and quickly.
|
||||
|
||||
## Fixing conflicts in the operation log
|
||||
|
||||
The log is applied in order from the last checkpoint.
|
||||
To stay in sync, the client regularly asks the server for the last commands.
|
||||
|
||||
When the log is applied, our system must enforce the following invariants:
|
||||
|
||||
- For all emails e1 and e2 in the log, such as e2.order > e1.order, then e2.uid > e1.uid
|
||||
|
||||
- For all emails e1 and e2 in the log, such as e1.uuid == e2.uuid, then e1.order == e2.order
|
||||
|
||||
If an invariant is broken, the conflict is solved with the following algorithm and the `uidvalidity` value is increased.
|
||||
|
||||
|
||||
```python
|
||||
def apply_mail_add(uuid, imap_uid):
|
||||
if imap_uid < internalseq:
|
||||
uidvalidity += internalseq - imap_uid
|
||||
mails.insert(uuid, internalseq, flags=["\Recent"])
|
||||
internalseq = internalseq + 1
|
||||
uidnext = internalseq
|
||||
|
||||
def apply_mail_del(uuid):
|
||||
mails.remove(uuid)
|
||||
internalseq = internalseq + 1
|
||||
```
|
||||
|
||||
A mathematical demonstration in Appendix D. shows that this algorithm indeed guarantees that under the same `uidvalidity`, different e-mails cannot share the same IMAP UID.
|
||||
|
||||
To illustrate, let us imagine two processes that have a first operation A in common, and then had a divergent state when one applied an operation B, and another one applied an operation C. For process 1, we have:
|
||||
|
||||
```python
|
||||
# state: uid-validity = 1, uid_next = 1, internalseq = 1
|
||||
(A) MAIL_ADD x 1
|
||||
# state: uid-validity = 1, x = 1, uid_next = 2, internalseq = 2
|
||||
(B) MAIL_ADD y 2
|
||||
# state: uid-validity = 1, x = 1, y = 2, uid_next = 3, internalseq = 3
|
||||
```
|
||||
|
||||
And for process 2 we have:
|
||||
|
||||
```python
|
||||
# state: uid-validity = 1, uid_next = 1, internalseq = 1
|
||||
(A) MAIL_ADD x 1
|
||||
# state: uid-validity = 1, x = 1, uid_next = 2, internalseq = 2
|
||||
(C) MAIL_ADD z 2
|
||||
# state: uid-validity = 1, x = 1, z = 2, uid_next = 3, internalseq = 3
|
||||
```
|
||||
|
||||
Suppose that a new client connects to one of the two processes after the conflicting operations have been communicated between them. They may have before connected either to process 1 or to process 2, so they might have observed either mail `y` or mail `z` with UID 2. The only way to make sure that the client will not be confused about mail UIDs is to bump the uidvalidity when the conflict is solved. This is indeed what happens with our algorithm: for both processes, once they have learned of the other's conflicting operation, they will execute the following set of operations and end in a deterministic state:
|
||||
|
||||
```python
|
||||
# state: uid-validity = 1, uid_next = 1, internalseq = 1
|
||||
(A) MAIL_ADD x 1
|
||||
# state: uid-validity = 1, x = 1, uid_next = 2, internalseq = 2
|
||||
(B) MAIL_ADD y 2
|
||||
# state: uid-validity = 1, x = 1, y = 2, uid_next = 3, internalseq = 3
|
||||
(C) MAIL_ADD z 2
|
||||
# conflict detected !
|
||||
# state: uid-validity = 2, x = 1, y = 2, z = 3, uid_next = 4, internalseq = 4
|
||||
```
|
||||
|
||||
## A computed state for efficient requests
|
||||
|
||||
From a data structure perspective, a list of commands is very inefficient to get the current state of the mailbox.
|
||||
Indeed, we don't want an `O(n)` complexity (where `n` is the number of log commands in the log) each time we want to know how many emails are stored in the mailbox.
|
||||
<!--To address this issue, we plan to maintain a locally computed (rollbackable) state of the mailbox.-->
|
||||
To address this issue, and thus query the mailbox efficiently, the MDA keeps an in-memory computed version of the logs, ie. the computed state.
|
||||
|
||||
**Mapping IMAP identifiers to email identifiers with B-Tree**
|
||||
|
||||
Core features of IMAP are synchronization and listing of emails.
|
||||
Its associated command is `FETCH`, it has 2 parameters, a range of `uid` (or `seq`) and a filter.
|
||||
For us, it means that we must be able to efficiently select a range of emails by their identifier, otherwise the user experience will be bad, and compute resources will be wasted.
|
||||
|
||||
We identified that by using an ordered map based on a B-Tree, we can satisfy this requirement in an optimal manner.
|
||||
For example, Rust defines a [BTreeMap](https://doc.rust-lang.org/std/collections/struct.BTreeMap.html) object in its standard library.
|
||||
We define the following structure for our mailbox:
|
||||
|
||||
```rust
|
||||
struct mailbox {
|
||||
emails: BTreeMap<ImapUid, (EmailUuid, Flags)>,
|
||||
flags: BTreeMap<Flag, BTreeMap<ImapUid, EmailUuid>>,
|
||||
name: String,
|
||||
uid_next: u32,
|
||||
uid_validity: u32,
|
||||
/* other fields */
|
||||
}
|
||||
```
|
||||
|
||||
This data structure allows us to efficiently select a range of emails by their identifier by walking the tree, allowing the server to be responsive to syncronisation request from clients.
|
||||
|
||||
**Checkpoints**
|
||||
|
||||
Having an in-memory computed state does not solve all the problems of operation on a log only, as 1) bootstrapping a fresh client is expensive as we have to replay possibly thousand of logs, and 2) logs would be kept indefinitely, wasting valuable storage resources.
|
||||
|
||||
As a solution to these limitations, the MDA regularly checkpoints the in-memory state. More specifically, it serializes it (eg. with MessagePack), compresses it (eg. with zstd), and then stores it on Garage through the S3 API.
|
||||
A fresh client would then only have to download the latest checkpoint and the range of logs between the checkpoint and now, allowing swift bootstraping while retaining all of the value of the log model.
|
||||
|
||||
Old logs and old checkpoints can be garbage collected after a few days for example as long as 1) the most recent checkpoint remains, 2) that all the logs after this checkpoint remain and 3) that we are confident enough that no log before this checkpoint will appear in the future.
|
||||
|
||||
|
|
@ -1,56 +0,0 @@
|
|||
# Mailboxes
|
||||
|
||||
IMAP servers, at their root, handle mailboxes.
|
||||
In this document, we explain the domain logic of IMAP and how we map it to Garage data
|
||||
with Aerogramme.
|
||||
|
||||
## IMAP Domain Logic
|
||||
|
||||
The main specification of IMAP is defined in [RFC3501](https://datatracker.ietf.org/doc/html/rfc3501).
|
||||
It defines 3 main objects: Mailboxes, Emails, and Flags. The following figure depicts how they work together:
|
||||
|
||||
|
||||
|
||||
Emails are stored ordered inside the mailbox, and for legacy reasons, the mailbox assigns 2 identifiers to each email we name `uid` and `seq`.
|
||||
|
||||
`seq` is the legacy identifier, it numbers messages in a sequence. Each time an email is deleted, the message numbering will change to keep a continuous sequence without holes.
|
||||
While this numbering is convenient for interactive operations, it is not efficient to synchronize mail locally and quickly detect missing new emails.
|
||||
|
||||
To solve this problem, `uid` identifiers were introduced later. They are monotonically increasing integers that must remain stable across time and sessions: when an email is deleted, its identifier is never reused.
|
||||
This is what Thunderbird uses for example when it synchronizes its mailboxes.
|
||||
|
||||
If this ordering cannot be kept, for example because two independent IMAP daemons were adding an email to the same mailbox at the same time, it is possible to change the ordering as long as we change a value named `uid-validity` to trigger a full resynchronization of all clients. As this operation is expensive, we want to minimize the probability of having to trigger a full resynchronization, but in practice, having this recovery mechanism simplifies the operation of an IMAP server by providing a rather simple solution to rare collision situations.
|
||||
|
||||
Flags are tags put on an email, some are defined at the protocol level, like `\Recent`, `\Deleted` or `\Seen`, which can be assigned or removed directly by the IMAP daemon.
|
||||
Others can be defined arbitrarily by the client, for which the MUA will apply its own logic.
|
||||
There is no mechanism in RFC3501 to synchronize flags between MUA besides listing the flags of all the emails.
|
||||
|
||||
IMAP has many extensions, such as [RFC5465](https://www.rfc-editor.org/rfc/rfc5465.html) or [RFC7162](https://datatracker.ietf.org/doc/html/rfc7162).
|
||||
They are referred to as capabilities and are [referenced by the IANA](https://www.iana.org/assignments/imap-capabilities/imap-capabilities.xhtml).
|
||||
For this project, we are aiming to implement only IMAP4rev1 and no extension at all.
|
||||
|
||||
|
||||
## Aerogramme Implementation
|
||||
|
||||
From a high-level perspective, we will handle _immutable_ emails differently from _mutable_ mailboxes and flags.
|
||||
Immutable data can be stored directly on Garage, as we do not fear reading an outdated value.
|
||||
For mutable data, we cannot store them directly in Garage.
|
||||
Instead, we choose to store a log of operations. Each client then applies this log of operation locally to rebuild its local state.
|
||||
|
||||
During this design phase, we noted that the S3 API semantic was too limited for us, so we introduced a second API, K2V, to have more flexibility.
|
||||
K2V is designed to store and fetch small values in batches, it uses 2 different keys: one to spread the data on the cluster (`P`), and one to sort linked data on the same node (`S`).
|
||||
Having data on the same node allows for more efficient queries among this data.
|
||||
|
||||
For performance reasons, we plan to introduce 2 optimizations.
|
||||
First, we store an email summary in K2V that allows fetching multiple entries at once.
|
||||
Second, we also store checkpoints of the logs in S3 to avoid keeping and replaying all the logs each time a client starts a session.
|
||||
We have the following data handled by Garage:
|
||||
|
||||
|
||||
|
||||
In Garage, it is important to carefully choose the key(s) that are used to store data to have fast queries, we propose the following model:
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 11 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 24 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 38 KiB |
|
|
@ -1,42 +0,0 @@
|
|||
# Notes
|
||||
|
||||
An IMAP trace extracted from Aerogramme:
|
||||
|
||||
```
|
||||
S: * OK Hello
|
||||
C: A1 LOGIN alan p455w0rd
|
||||
S: A1 OK Completed
|
||||
C: A2 SELECT INBOX
|
||||
S: * 0 EXISTS
|
||||
S: * 0 RECENT
|
||||
S: * FLAGS (\Seen \Answered \Flagged \Deleted \Draft)
|
||||
S: * OK [PERMANENTFLAGS (\Seen \Answered \Flagged \Deleted \Draft \*)] Flags permitted
|
||||
S: * OK [UIDVALIDITY 1] UIDs valid
|
||||
S: * OK [UIDNEXT 1] Predict next UID
|
||||
S: A2 OK [READ-WRITE] Select completed
|
||||
C: A3 NOOP
|
||||
S: A3 OK NOOP completed.
|
||||
<---- e-mail arrives through LMTP server ---->
|
||||
C: A4 NOOP
|
||||
S: * 1 EXISTS
|
||||
S: A4 OK NOOP completed.
|
||||
C: A5 FETCH 1 FULL
|
||||
S: * 1 FETCH (UID 1 FLAGS () INTERNALDATE "06-Jul-2022 14:46:42 +0000"
|
||||
RFC822.SIZE 117 ENVELOPE (NIL "test" (("Alan Smith" NIL "alan" "smith.me"))
|
||||
NIL NIL (("Alan Smith" NIL "alan" "aerogramme.tld")) NIL NIL NIL NIL)
|
||||
BODY ("TEXT" "test" NIL "test" "test" "test" 1 1))
|
||||
S: A5 OK FETCH completed
|
||||
C: A6 FETCH 1 (RFC822)
|
||||
S: * 1 FETCH (UID 1 RFC822 {117}
|
||||
S: Subject: test
|
||||
S: From: Alan Smith <alan@smith.me>
|
||||
S: To: Alan Smith <alan@aerogramme.tld>
|
||||
S:
|
||||
S: Hello, world!
|
||||
S: .
|
||||
S: )
|
||||
S: A6 OK FETCH completed
|
||||
C: A7 LOGOUT
|
||||
S: * BYE Logging out
|
||||
S: A7 OK Logout completed
|
||||
```
|
||||
|
|
@ -1,61 +0,0 @@
|
|||
# Overview
|
||||
|
||||
Aérogramme stands at the interface between the Garage storage server, and the user's e-mail client. It provides regular IMAP access on the client-side, and stores encrypted e-mail data on the server-side. Aérogramme also provides an LMTP server interface through which incoming mail can be forwarded by the MTA (e.g. Postfix).
|
||||
|
||||
<center>
|
||||
<img src="./aero-compo.png" alt="Aerogramme components"/>
|
||||
<br>
|
||||
<i>Figure 1: Aérogramme, our IMAP daemon, stores its data encrypted in Garage and provides regular IMAP access to mail clients</i></center>
|
||||
|
||||
|
||||
**Overview of architecture**
|
||||
|
||||
Figure 2 below shows an overview of Aérogramme's architecture. Each user has a personal Garage bucket in which to store their mailbox contents. We will document below the details of the components that make up Aérogramme, but let us first provide a high-level overview. The two main classes, `User` and `Mailbox`, define how data is stored in this bucket, and provide a high-level interface with primitives such as reading the message index, loading a mail's content, copying, moving, and deleting messages, etc. This mail storage system is supported by two important primitives: a cryptography management system that provides encryption keys for user's data, and a simple log-like database system inspired by Bayou [1] which we have called Bay, that we use to store the index of messages in each mailbox. The mail storage system is made accessible to the outside world by two subsystems: an LMTP server that allows for incoming mail to be received and stored in a user's bucket, in a staging area, and the IMAP server itself which allows full-fledged manipulation of mailbox data by users.
|
||||
|
||||
<center>
|
||||
<img src="./aero-schema.png" alt="Aerogramme internals"/>
|
||||
<i>Figure 2: Overview of Aérogramme's architecture and internal data structures for a given user, Alice</i></center>
|
||||
|
||||
|
||||
**Cryptography**
|
||||
|
||||
Our cryptography module is taking care of: authenticating users against a data source (using their IMAP login and password), returning a set of credentials that allow read/write access to a Garage bucket, as well as a set of secret encryption keys used to encrypt and decrypt data stored in the bucket.
|
||||
The cryptography module makes use of the user's authentication password as a passphrase to decrypt the user's secret keys, which are stored in the user's bucket in a dedicated K2V section.
|
||||
|
||||
This module can use either of two data sources for user authentication:
|
||||
|
||||
- LDAP, in which case the password (which is also the passphrase for decrypting the user's secret keys) must match the LDAP password of the user.
|
||||
- Static, in which case the users are statically declared in Aérogramme's configuration file, and can have any password.
|
||||
|
||||
The static authentication source can be used in a deployment scenario shown in Figure 3, where Aérogramme is not running on the side of the service provider, but on the user's device itself. In this case, the user can use any password to encrypt their data in the bucket; the only credentials they need for authentication against the service provider are the S3 and K2V API access keys.
|
||||
|
||||
<center>
|
||||
<img src="./aero-paranoid.png" alt="user side encryption" />
|
||||
<br>
|
||||
<i>Figure 3: alternative deployment of Aérogramme on the user's device: the service provider never gets access to the plaintext data.</i></center>
|
||||
|
||||
The cryptography module also has a "public authentication" method, which allows the LMTP module to retrieve only a public key for the user to write incoming messages to the user's bucket but without having access to all of the existing encrypted data.
|
||||
|
||||
The cryptography module of Aérogramme is based on standard cryptographic primitives from `libsodium` and follows best practices in the domain.
|
||||
|
||||
**Bay, a simplification of Bayou**
|
||||
|
||||
In our last milestone report, we described how we intended to implement the message index for IMAP mailboxes, based on an eventually-consistent log-like data structure. The principles of this system have been established in Bayou in 1995 [1], allowing users to use a weakly-coordinated datastore to exchange data and solve write conflicts. Bayou is based on a sequential specification, which defines the action that operations in the log have on the shared object's state. To handle concurrent modification, Bayou allows for log entries to be appended in non-sequential order: in case a process reads a log entry that was written earlier by another process, it can rewind its execution of the sequential specification to the point where the newly acquired operation should have been executed, and then execute the log again starting from this point. The challenge then consists in defining a sequential specification that provides the desired semantics for the application. In our last milestone report (milestone 3.A), we described a sequential specification that solves the UID assignment problem in IMAP and proved it correct. We refer the reader to that document for more details.
|
||||
|
||||
For milestone 3B, we have implemented our customized version of Bayou, which we call Bay. Bay implements the log-like semantics and the rewind ability of Bayou, however, it makes use of a much simpler data system: Bay is not operating on a relational database that is stored on disk, but simply on a data structure in RAM, for which a full checkpoint is written regularly. We decided against using a complex database as we observed that the expected size of the data structures we would be handling (the message indexes for each mailbox) wouldn't be so big most of the time, and having a full copy in RAM was perfectly acceptable. This allows for a drastic simplification in comparison to the proposal of the original Bayou paper [1]. On the other side, we added encryption in Bay so that both log entries and checkpoints are stored encrypted in Garage using the user's secret key, meaning that a malicious Garage administrator cannot read the content of a user's mailbox index.
|
||||
|
||||
**LMTP server and incoming mail handler**
|
||||
|
||||
To handle incoming mail, we had to add a simple LMTP server to Aérogramme. This server uses the public authentication method of the cryptography module to retrieve a set of public credentials (in particular, a public key for asymmetric encryption) for storing incoming messages. The incoming messages are stored in their raw RFC822 form (encrypted) in a specific folder of the Garage bucket called `incoming/`. When a user logs in with their username and password, at which time Aérogramme can decrypt the user's secret keys, a special process is launched that watches the incoming folder and moves these messages to the `INBOX` folder. This task can only be done by a process that knows the user's secret keys, as it has to modify the mailbox index of the `INBOX` folder, which is encrypted using the user's secret keys. In later versions of Aérogramme, this process would be the perfect place to implement mail filtering logic using user-specified rules. These rules could be stored in a dedicated section of the bucket, again encrypted with the user's secret keys.
|
||||
|
||||
To implement the LMTP server, we chose to make use of the `smtp-server` crate from the [Kannader](https://github.com/Ekleog/kannader) project (an MTA written in Rust). The `smtp-server` crate had all of the necessary functionality for building SMTP servers, however, it did not handle LMTP. As LMTP is extremely close to SMTP, we were able to extend the `smtp-server` module to allow it to be used for the implementation of both SMTP and LMTP servers. Our work has been proposed as a [pull request](https://github.com/Ekleog/kannader/pull/178) to be merged back upstream in Kannader, which should be integrated soon.
|
||||
|
||||
**IMAP server**
|
||||
|
||||
The last part that remains to build Aérogramme is to implement the logic behind the IMAP protocol and to link it with the mail storage primitives. We started by implementing a state machine that handled the transitions between the different states in the IMAP protocol: ANONYMOUS (before login), AUTHENTICATED (after login), and SELECTED (once a mailbox has been selected for reading/writing). In the SELECTED state, the IMAP session is linked to a given mailbox of the user. In addition, the IMAP server has to keep track of which updates to the mailbox it has sent (or not) to the client so that it can produce IMAP messages consistent with what the client believes to be in the mailbox. In particular, many IMAP commands make use of mail sequence numbers to identify messages, which are indices in the sorted array of all of the messages in the mailbox. However, if messages are added or removed concurrently, these sequence numbers change: hence we must keep a snapshot of the mailbox's index *as the client knows it*, which is not necessarily the same as what is _actually_ in the mailbox, to generate messages that the client will understand correctly. This snapshot is called a *mailbox view* and is synced regularly with the actual mailbox, at which time the corresponding IMAP updates are sent. This can be done only at specific moments when permitted by the IMAP protocol.
|
||||
|
||||
The second part of this task consisted in implementing all of the IMAP protocol commands. Most are relatively straightforward, however, one command, in particular, needed special care: the FETCH command. The FETCH command in the IMAP protocol can return the contents of a message to the client. However, it must also understand precisely the semantics of the content of an e-mail message, as the client can specify very precisely how the message should be returned. For instance, in the case of a multipart message with attachments, the client can emit a FECTH command requesting only a certain attachment of the message to be returned, and not the whole message. To implement such semantics, we have based ourselves on the [`mail-parser`](https://docs.rs/mail-parser/latest/mail_parser/) crate, which can fully parse an RFC822-formatted e-mail message, and also supports some extensions such as MIME. To validate that we were correctly converting the parsed message structure to IMAP messages, we designed a test suite composed of several weirdly shaped e-mail messages, whose IMAP structure definition we extracted by taking Dovecot as a reference. We were then able to compare the output of Aérogramme on these messages with the reference consisting in what was returned by Dovecot.
|
||||
|
||||
## References
|
||||
|
||||
- [1] Terry, D. B., Theimer, M. M., Petersen, K., Demers, A. J., Spreitzer, M. J., & Hauser, C. H. (1995). Managing update conflicts in Bayou, a weakly connected replicated storage system. *ACM SIGOPS Operating Systems Review*, 29(5), 172-182. ([PDF](https://dl.acm.org/doi/pdf/10.1145/224057.224070))
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
# RFC coverage
|
||||
|
||||
*Not yet written*
|
||||
|
|
@ -1,90 +0,0 @@
|
|||
# Setup
|
||||
|
||||
You must start by creating a user profile in Garage. Run the following command after adjusting the parameters to your configuration:
|
||||
|
||||
```bash
|
||||
cargo run -- first-login \
|
||||
--region garage \
|
||||
--k2v-endpoint http://127.0.0.1:3904 \
|
||||
--s3-endpoint http://127.0.0.1:3900 \
|
||||
--aws-access-key-id GK... \
|
||||
--aws-secret-access-key c0ffee... \
|
||||
--bucket mailrage-me \
|
||||
--user-secret s3cr3t
|
||||
```
|
||||
|
||||
*Note: user-secret is not the user's password. It is an additional secret used when deriving user's secret key from their password. The idea is that, even if user leaks their password, their encrypted data remain safe as long as this additional secret does not leak. You can generate it with openssl for example: `openssl rand -base64 30`. Read [Cryptography & key management](./crypt-key.md) for more details.*
|
||||
|
||||
|
||||
The program will interactively ask you some questions and finally generates for you a snippet of configuration:
|
||||
|
||||
```
|
||||
Please enter your password for key decryption.
|
||||
If you are using LDAP login, this must be your LDAP password.
|
||||
If you are using the static login provider, enter any password, and this will also become your password for local IMAP access.
|
||||
Enter password:
|
||||
Confirm password:
|
||||
|
||||
Cryptographic key setup is complete.
|
||||
|
||||
If you are using the static login provider, add the following section to your .toml configuration file:
|
||||
|
||||
[login_static.users.<username>]
|
||||
password = "$argon2id$v=19$m=4096,t=3,p=1$..."
|
||||
aws_access_key_id = "GK..."
|
||||
aws_secret_access_key = "c0ffee..."
|
||||
```
|
||||
|
||||
In this tutorial, we will use the static login provider (and not the LDAP one).
|
||||
We will thus create a config file named `aerogramme.toml` in which we will paste the previous snippet. You also need to enter some other keys. In the end, your file should look like that:
|
||||
|
||||
```toml
|
||||
s3_endpoint = "http://127.0.0.1:3900"
|
||||
k2v_endpoint = "http://127.0.0.1:3904"
|
||||
aws_region = "garage"
|
||||
|
||||
[lmtp]
|
||||
bind_addr = "[::1]:12024"
|
||||
hostname = "aerogramme.tld"
|
||||
|
||||
[imap]
|
||||
bind_addr = "[::1]:1993"
|
||||
|
||||
[login_static]
|
||||
default_bucket = "mailrage"
|
||||
|
||||
[login_static.users.me]
|
||||
bucket = "mailrage-me"
|
||||
user_secret = "s3cr3t"
|
||||
email_addresses = [
|
||||
"me@aerogramme.tld"
|
||||
]
|
||||
|
||||
# copy pasted values from first-login
|
||||
password = "$argon2id$v=19$m=4096,t=3,p=1$..."
|
||||
aws_access_key_id = "GK..."
|
||||
aws_secret_access_key = "c0ffee..."
|
||||
```
|
||||
|
||||
If you fear to loose your password, you can backup your key with the following command:
|
||||
|
||||
```bash
|
||||
cargo run -- show-keys \
|
||||
--region garage \
|
||||
--k2v-endpoint http://127.0.0.1:3904 \
|
||||
--s3-endpoint http://127.0.0.1:3900 \
|
||||
--aws-access-key-id GK... \
|
||||
--aws-secret-access-key c0ffee... \
|
||||
--bucket mailrage-me \
|
||||
--user-secret s3cr3t
|
||||
```
|
||||
|
||||
You will then be asked for your key decryption password:
|
||||
|
||||
```
|
||||
Enter key decryption password:
|
||||
master_key = "..."
|
||||
secret_key = "..."
|
||||
```
|
||||
|
||||
You are now ready to [validate your installation](./validate.md).
|
||||
|
|
@ -1,40 +0,0 @@
|
|||
# Validate
|
||||
|
||||
Start a server as follow:
|
||||
|
||||
```bash
|
||||
cargo run -- server
|
||||
```
|
||||
|
||||
Inject emails:
|
||||
|
||||
```bash
|
||||
./test/inject_emails.sh '<me@aerogramme.tld>' dxflrs
|
||||
```
|
||||
|
||||
Now you can connect your mailbox with `mutt`.
|
||||
Start by creating a config file, for example we used the following `~/.muttrc` file:
|
||||
|
||||
```ini
|
||||
set imap_user = quentin
|
||||
set imap_pass = p455w0rd
|
||||
set folder = imap://localhost:1993
|
||||
set spoolfile = +INBOX
|
||||
set ssl_starttls = no
|
||||
set ssl_force_tls = no
|
||||
mailboxes = +INBOX
|
||||
bind index G imap-fetch-mail
|
||||
```
|
||||
|
||||
And then simply launch `mutt`.
|
||||
The first time nothing will happen as Aerogramme must
|
||||
process your incoming emails. Just ask `mutt` to refresh its
|
||||
view by pressing `G` (for *Get*).
|
||||
|
||||
Now, you should see some emails:
|
||||
|
||||
|
||||
|
||||
And you can read them:
|
||||
|
||||
|
||||
278
flake.lock
278
flake.lock
|
|
@ -1,278 +0,0 @@
|
|||
{
|
||||
"nodes": {
|
||||
"albatros": {
|
||||
"inputs": {
|
||||
"flake-utils": "flake-utils",
|
||||
"nixpkgs": "nixpkgs"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1684830446,
|
||||
"narHash": "sha256-jyYwYYNKSe40Y9OirIkeFTvTvqNj0NErh4TNBJmujw4=",
|
||||
"ref": "main",
|
||||
"rev": "fb80c5d6734044ca7718989a3b36503b9463f1b2",
|
||||
"revCount": 81,
|
||||
"type": "git",
|
||||
"url": "https://git.deuxfleurs.fr/Deuxfleurs/albatros.git"
|
||||
},
|
||||
"original": {
|
||||
"ref": "main",
|
||||
"type": "git",
|
||||
"url": "https://git.deuxfleurs.fr/Deuxfleurs/albatros.git"
|
||||
}
|
||||
},
|
||||
"cargo2nix": {
|
||||
"inputs": {
|
||||
"flake-compat": "flake-compat",
|
||||
"flake-utils": "flake-utils_2",
|
||||
"nixpkgs": "nixpkgs_2",
|
||||
"rust-overlay": "rust-overlay"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1673262828,
|
||||
"narHash": "sha256-pDqno5/2ghQDt4LjVt5ZUMV9pTSA5rGGdz6Skf2rBwc=",
|
||||
"owner": "Alexis211",
|
||||
"repo": "cargo2nix",
|
||||
"rev": "505caa32110d42ee03bd68b47031142eff9c827b",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "Alexis211",
|
||||
"ref": "custom_unstable",
|
||||
"repo": "cargo2nix",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"fenix": {
|
||||
"inputs": {
|
||||
"nixpkgs": "nixpkgs_3",
|
||||
"rust-analyzer-src": "rust-analyzer-src"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1708669354,
|
||||
"narHash": "sha256-eGhZLjF59aF9bYdSOleT1BD94qvo1NgMio4vMKBzxgY=",
|
||||
"owner": "nix-community",
|
||||
"repo": "fenix",
|
||||
"rev": "a0f0f781683e4e93b61beaf1dfee4dd34cf3a092",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-community",
|
||||
"repo": "fenix",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"flake-compat": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
"lastModified": 1668681692,
|
||||
"narHash": "sha256-Ht91NGdewz8IQLtWZ9LCeNXMSXHUss+9COoqu6JLmXU=",
|
||||
"owner": "edolstra",
|
||||
"repo": "flake-compat",
|
||||
"rev": "009399224d5e398d03b22badca40a37ac85412a1",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "edolstra",
|
||||
"repo": "flake-compat",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"flake-utils": {
|
||||
"inputs": {
|
||||
"systems": "systems"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1681202837,
|
||||
"narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=",
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"rev": "cfacdce06f30d2b68473a46042957675eebb3401",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"flake-utils_2": {
|
||||
"locked": {
|
||||
"lastModified": 1667395993,
|
||||
"narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"flake-utils_3": {
|
||||
"inputs": {
|
||||
"systems": "systems_2"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1705309234,
|
||||
"narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=",
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1678964307,
|
||||
"narHash": "sha256-POV15raLJzwns6U84W4aWNSeSJRXTz7xWQW6IcrWQns=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "fd4f7832961053e6095af8de8d6a57b5ad402f19",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs_2": {
|
||||
"locked": {
|
||||
"lastModified": 1672580127,
|
||||
"narHash": "sha256-3lW3xZslREhJogoOkjeZtlBtvFMyxHku7I/9IVehhT8=",
|
||||
"owner": "nixos",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "0874168639713f547c05947c76124f78441ea46c",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nixos",
|
||||
"ref": "release-22.05",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs_3": {
|
||||
"locked": {
|
||||
"lastModified": 1706550542,
|
||||
"narHash": "sha256-UcsnCG6wx++23yeER4Hg18CXWbgNpqNXcHIo5/1Y+hc=",
|
||||
"owner": "nixos",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "97b17f32362e475016f942bbdfda4a4a72a8a652",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nixos",
|
||||
"ref": "nixos-unstable",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs_4": {
|
||||
"locked": {
|
||||
"lastModified": 1708673722,
|
||||
"narHash": "sha256-FPbPhA727wuVkmR21Va6scRjAmj4pk3U8blteaXB/Hg=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "92cf4feb2b9091466a82b27e4bb045cbccc2ba09",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"ref": "master",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"root": {
|
||||
"inputs": {
|
||||
"albatros": "albatros",
|
||||
"cargo2nix": "cargo2nix",
|
||||
"fenix": "fenix",
|
||||
"flake-utils": "flake-utils_3",
|
||||
"nixpkgs": "nixpkgs_4"
|
||||
}
|
||||
},
|
||||
"rust-analyzer-src": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
"lastModified": 1706735270,
|
||||
"narHash": "sha256-IJk+UitcJsxzMQWm9pa1ZbJBriQ4ginXOlPyVq+Cu40=",
|
||||
"owner": "rust-lang",
|
||||
"repo": "rust-analyzer",
|
||||
"rev": "42cb1a2bd79af321b0cc503d2960b73f34e2f92b",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "rust-lang",
|
||||
"ref": "nightly",
|
||||
"repo": "rust-analyzer",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"rust-overlay": {
|
||||
"inputs": {
|
||||
"flake-utils": [
|
||||
"cargo2nix",
|
||||
"flake-utils"
|
||||
],
|
||||
"nixpkgs": [
|
||||
"cargo2nix",
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1673231106,
|
||||
"narHash": "sha256-Tbw4N/TL+nHmxF8RBoOJbl/6DRRzado/9/ttPEzkGr8=",
|
||||
"owner": "oxalica",
|
||||
"repo": "rust-overlay",
|
||||
"rev": "3488cec01351c2f1086b02a3a61808be7a25103e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "oxalica",
|
||||
"repo": "rust-overlay",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"systems": {
|
||||
"locked": {
|
||||
"lastModified": 1681028828,
|
||||
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"systems_2": {
|
||||
"locked": {
|
||||
"lastModified": 1681028828,
|
||||
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"type": "github"
|
||||
}
|
||||
}
|
||||
},
|
||||
"root": "root",
|
||||
"version": 7
|
||||
}
|
||||
233
flake.nix
233
flake.nix
|
|
@ -1,233 +0,0 @@
|
|||
{
|
||||
description = "Aerogramme";
|
||||
|
||||
inputs = {
|
||||
nixpkgs.url = "github:NixOS/nixpkgs/master";
|
||||
flake-utils.url = "github:numtide/flake-utils";
|
||||
|
||||
# this patched version of cargo2nix makes easier to use clippy for building
|
||||
cargo2nix = {
|
||||
type = "github";
|
||||
owner = "Alexis211";
|
||||
repo = "cargo2nix";
|
||||
ref = "custom_unstable";
|
||||
};
|
||||
|
||||
# use rust project builds
|
||||
fenix.url = "github:nix-community/fenix";
|
||||
|
||||
# import alba releasing tool
|
||||
albatros.url = "git+https://git.deuxfleurs.fr/Deuxfleurs/albatros.git?ref=main";
|
||||
};
|
||||
|
||||
outputs = { self, nixpkgs, cargo2nix, flake-utils, fenix, albatros }:
|
||||
let platformArtifacts = flake-utils.lib.eachSystem [
|
||||
"x86_64-unknown-linux-musl"
|
||||
"aarch64-unknown-linux-musl"
|
||||
"armv6l-unknown-linux-musleabihf"
|
||||
] (targetHost: let
|
||||
|
||||
# with fenix, we get builds from the rust project.
|
||||
# they are done with an old version of musl (prior to 1.2.x that is used in NixOS),
|
||||
# however musl has a breaking change from 1.1.x to 1.2.x on 32 bit systems.
|
||||
# so we pin the lib to 1.1.x to avoid recompiling rust ourselves.
|
||||
muslOverlay = self: super: {
|
||||
musl = super.musl.overrideAttrs(old: if targetHost == "armv6l-unknown-linux-musleabihf" then rec {
|
||||
pname = "musl";
|
||||
version = "1.1.24";
|
||||
src = builtins.fetchurl {
|
||||
url = "https://musl.libc.org/releases/${pname}-${version}.tar.gz";
|
||||
sha256 = "sha256:18r2a00k82hz0mqdvgm7crzc7305l36109c0j9yjmkxj2alcjw0k";
|
||||
};
|
||||
} else {});
|
||||
};
|
||||
|
||||
pkgs = import nixpkgs {
|
||||
system = "x86_64-linux"; # hardcoded as we will cross compile
|
||||
crossSystem = {
|
||||
config = targetHost; # here we cross compile
|
||||
isStatic = true;
|
||||
};
|
||||
overlays = [
|
||||
cargo2nix.overlays.default
|
||||
muslOverlay
|
||||
];
|
||||
};
|
||||
|
||||
rustTarget = if targetHost == "armv6l-unknown-linux-musleabihf" then "arm-unknown-linux-musleabihf" else targetHost;
|
||||
|
||||
# release builds
|
||||
rustRelease = pkgs.rustBuilder.makePackageSet({
|
||||
packageFun = import ./Cargo.nix;
|
||||
target = rustTarget;
|
||||
release = true;
|
||||
#rustcLinkFlags = [ "--cfg" "tokio_unstable" ];
|
||||
#rustcBuildFlags = [ "--cfg" "tokio_unstable" ];
|
||||
rustToolchain = with fenix.packages.x86_64-linux; combine [
|
||||
minimal.cargo
|
||||
minimal.rustc
|
||||
targets.${rustTarget}.latest.rust-std
|
||||
];
|
||||
});
|
||||
|
||||
# debug builds with clippy as the compiler (hack to speed up compilation)
|
||||
debugBuildEnv = (drv:
|
||||
''
|
||||
${drv.setBuildEnv or ""}
|
||||
echo
|
||||
echo --- BUILDING WITH CLIPPY ---
|
||||
echo
|
||||
|
||||
export NIX_RUST_BUILD_FLAGS="''${NIX_RUST_BUILD_FLAGS} --deny warnings"
|
||||
export NIX_RUST_LINK_FLAGS="''${NIX_RUST_LINK_FLAGS} --deny warnings"
|
||||
export RUSTC="''${CLIPPY_DRIVER}"
|
||||
'');
|
||||
|
||||
rustDebug = pkgs.rustBuilder.makePackageSet({
|
||||
packageFun = import ./Cargo.nix;
|
||||
target = rustTarget;
|
||||
release = false;
|
||||
rustToolchain = with fenix.packages.x86_64-linux; combine [
|
||||
default.cargo
|
||||
default.rustc
|
||||
default.clippy
|
||||
targets.${rustTarget}.latest.rust-std
|
||||
];
|
||||
packageOverrides = pkgs: pkgs.rustBuilder.overrides.all ++ [
|
||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||
name = "aerogramme";
|
||||
overrideAttrs = drv: {
|
||||
setBuildEnv = (debugBuildEnv drv);
|
||||
};
|
||||
})
|
||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||
name = "smtp-message";
|
||||
overrideAttrs = drv: {
|
||||
/*setBuildEnv = (traceBuildEnv drv);
|
||||
propagatedBuildInputs = drv.propagatedBuildInputs or [ ] ++ [
|
||||
traceRust
|
||||
];*/
|
||||
};
|
||||
})
|
||||
];
|
||||
});
|
||||
|
||||
|
||||
crate = (rustRelease.workspace.aerogramme {});
|
||||
|
||||
# binary extract
|
||||
bin = pkgs.stdenv.mkDerivation {
|
||||
pname = "${crate.name}-bin";
|
||||
version = crate.version;
|
||||
dontUnpack = true;
|
||||
dontBuild = true;
|
||||
installPhase = ''
|
||||
cp ${crate.bin}/bin/aerogramme $out
|
||||
'';
|
||||
};
|
||||
|
||||
# fhs extract
|
||||
fhs = pkgs.stdenv.mkDerivation {
|
||||
pname = "${crate.name}-fhs";
|
||||
version = crate.version;
|
||||
dontUnpack = true;
|
||||
dontBuild = true;
|
||||
installPhase = ''
|
||||
mkdir -p $out/bin
|
||||
cp ${crate.bin}/bin/aerogramme $out/bin/
|
||||
'';
|
||||
};
|
||||
|
||||
# docker packaging
|
||||
archMap = {
|
||||
"x86_64-unknown-linux-musl" = {
|
||||
GOARCH = "amd64";
|
||||
};
|
||||
"aarch64-unknown-linux-musl" = {
|
||||
GOARCH = "arm64";
|
||||
};
|
||||
"armv6l-unknown-linux-musleabihf" = {
|
||||
GOARCH = "arm";
|
||||
};
|
||||
};
|
||||
container = pkgs.dockerTools.buildImage {
|
||||
name = "dxflrs/aerogramme";
|
||||
architecture = (builtins.getAttr targetHost archMap).GOARCH;
|
||||
copyToRoot = fhs;
|
||||
config = {
|
||||
Env = [ "PATH=/bin" ];
|
||||
Cmd = [ "aerogramme" "--dev" "provider" "daemon" ];
|
||||
};
|
||||
};
|
||||
|
||||
in {
|
||||
meta = {
|
||||
version = crate.version;
|
||||
};
|
||||
packages = {
|
||||
inherit fhs container;
|
||||
debug = (rustDebug.workspace.aerogramme {}).bin;
|
||||
aerogramme = bin;
|
||||
default = self.packages.${targetHost}.aerogramme;
|
||||
};
|
||||
});
|
||||
|
||||
###
|
||||
#
|
||||
# RELEASE STUFF
|
||||
#
|
||||
###
|
||||
gpkgs = import nixpkgs {
|
||||
system = "x86_64-linux"; # hardcoded as we will cross compile
|
||||
};
|
||||
alba = albatros.packages.x86_64-linux.alba;
|
||||
|
||||
# Shell
|
||||
shell = gpkgs.mkShell {
|
||||
buildInputs = [
|
||||
cargo2nix.packages.x86_64-linux.default
|
||||
fenix.packages.x86_64-linux.minimal.toolchain
|
||||
fenix.packages.x86_64-linux.rust-analyzer
|
||||
];
|
||||
shellHook = ''
|
||||
echo "AEROGRAME DEVELOPMENT SHELL ${fenix.packages.x86_64-linux.minimal.rustc}"
|
||||
export RUST_SRC_PATH="${fenix.packages.x86_64-linux.latest.rust-src}/lib/rustlib/src/rust/library"
|
||||
export RUST_ANALYZER_INTERNALS_DO_NOT_USE='this is unstable'
|
||||
'';
|
||||
};
|
||||
|
||||
# Used only to fetch the "version"
|
||||
version = platformArtifacts.meta.x86_64-unknown-linux-musl.version;
|
||||
|
||||
build = gpkgs.writeScriptBin "aerogramme-build" ''
|
||||
set -euxo pipefail
|
||||
|
||||
# static
|
||||
nix build --print-build-logs .#packages.x86_64-unknown-linux-musl.aerogramme -o static/linux/amd64/aerogramme
|
||||
nix build --print-build-logs .#packages.aarch64-unknown-linux-musl.aerogramme -o static/linux/arm64/aerogramme
|
||||
nix build --print-build-logs .#packages.armv6l-unknown-linux-musleabihf.aerogramme -o static/linux/arm/aerogramme
|
||||
|
||||
# containers
|
||||
nix build --print-build-logs .#packages.x86_64-unknown-linux-musl.container -o docker/linux.amd64.tar.gz
|
||||
nix build --print-build-logs .#packages.aarch64-unknown-linux-musl.container -o docker/linux.arm64.tar.gz
|
||||
nix build --print-build-logs .#packages.armv6l-unknown-linux-musleabihf.container -o docker/linux.arm.tar.gz
|
||||
'';
|
||||
|
||||
push = gpkgs.writeScriptBin "aerogramme-publish" ''
|
||||
set -euxo pipefail
|
||||
|
||||
${alba} static push -t aerogramme:${version} static/ 's3://download.deuxfleurs.org?endpoint=garage.deuxfleurs.fr&s3ForcePathStyle=true®ion=garage' 1>&2
|
||||
${alba} container push -t aerogramme:${version} docker/ 's3://registry.deuxfleurs.org?endpoint=garage.deuxfleurs.fr&s3ForcePathStyle=true®ion=garage' 1>&2
|
||||
${alba} container push -t aerogramme:${version} docker/ "docker://docker.io/dxflrs/aerogramme:${version}" 1>&2
|
||||
'';
|
||||
|
||||
in
|
||||
{
|
||||
devShells.x86_64-linux.default = shell;
|
||||
packages = {
|
||||
x86_64-linux = {
|
||||
inherit build push;
|
||||
};
|
||||
} // platformArtifacts.packages;
|
||||
};
|
||||
}
|
||||
941
src/auth.rs
941
src/auth.rs
|
|
@ -1,941 +0,0 @@
|
|||
use std::net::SocketAddr;
|
||||
|
||||
use anyhow::{anyhow, bail, Result};
|
||||
use futures::stream::{FuturesUnordered, StreamExt};
|
||||
use tokio::io::BufStream;
|
||||
use tokio::io::{AsyncBufReadExt, AsyncWriteExt};
|
||||
use tokio::net::{TcpListener, TcpStream};
|
||||
use tokio::sync::watch;
|
||||
|
||||
use crate::config::AuthConfig;
|
||||
use crate::login::ArcLoginProvider;
|
||||
|
||||
/// Seek compatibility with the Dovecot Authentication Protocol
|
||||
///
|
||||
/// ## Trace
|
||||
///
|
||||
/// ```text
|
||||
/// S: VERSION 1 2
|
||||
/// S: MECH PLAIN plaintext
|
||||
/// S: MECH LOGIN plaintext
|
||||
/// S: SPID 15
|
||||
/// S: CUID 17654
|
||||
/// S: COOKIE f56692bee41f471ed01bd83520025305
|
||||
/// S: DONE
|
||||
/// C: VERSION 1 2
|
||||
/// C: CPID 1
|
||||
///
|
||||
/// C: AUTH 2 PLAIN service=smtp
|
||||
/// S: CONT 2
|
||||
/// C: CONT 2 base64stringFollowingRFC4616==
|
||||
/// S: OK 2 user=alice@example.tld
|
||||
///
|
||||
/// C: AUTH 42 LOGIN service=smtp
|
||||
/// S: CONT 42 VXNlcm5hbWU6
|
||||
/// C: CONT 42 b64User
|
||||
/// S: CONT 42 UGFzc3dvcmQ6
|
||||
/// C: CONT 42 b64Pass
|
||||
/// S: FAIL 42 user=alice
|
||||
/// ```
|
||||
///
|
||||
/// ## RFC References
|
||||
///
|
||||
/// PLAIN SASL - https://datatracker.ietf.org/doc/html/rfc4616
|
||||
///
|
||||
///
|
||||
/// ## Dovecot References
|
||||
///
|
||||
/// https://doc.dovecot.org/developer_manual/design/auth_protocol/
|
||||
/// https://doc.dovecot.org/configuration_manual/authentication/authentication_mechanisms/#authentication-authentication-mechanisms
|
||||
/// https://doc.dovecot.org/configuration_manual/howto/simple_virtual_install/#simple-virtual-install-smtp-auth
|
||||
/// https://doc.dovecot.org/configuration_manual/howto/postfix_and_dovecot_sasl/#howto-postfix-and-dovecot-sasl
|
||||
pub struct AuthServer {
|
||||
login_provider: ArcLoginProvider,
|
||||
bind_addr: SocketAddr,
|
||||
}
|
||||
|
||||
impl AuthServer {
|
||||
pub fn new(config: AuthConfig, login_provider: ArcLoginProvider) -> Self {
|
||||
Self {
|
||||
bind_addr: config.bind_addr,
|
||||
login_provider,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn run(self: Self, mut must_exit: watch::Receiver<bool>) -> Result<()> {
|
||||
let tcp = TcpListener::bind(self.bind_addr).await?;
|
||||
tracing::info!(
|
||||
"SASL Authentication Protocol listening on {:#}",
|
||||
self.bind_addr
|
||||
);
|
||||
|
||||
let mut connections = FuturesUnordered::new();
|
||||
|
||||
while !*must_exit.borrow() {
|
||||
let wait_conn_finished = async {
|
||||
if connections.is_empty() {
|
||||
futures::future::pending().await
|
||||
} else {
|
||||
connections.next().await
|
||||
}
|
||||
};
|
||||
|
||||
let (socket, remote_addr) = tokio::select! {
|
||||
a = tcp.accept() => a?,
|
||||
_ = wait_conn_finished => continue,
|
||||
_ = must_exit.changed() => continue,
|
||||
};
|
||||
|
||||
tracing::info!("AUTH: accepted connection from {}", remote_addr);
|
||||
let conn = tokio::spawn(
|
||||
NetLoop::new(socket, self.login_provider.clone(), must_exit.clone()).run_error(),
|
||||
);
|
||||
|
||||
connections.push(conn);
|
||||
}
|
||||
drop(tcp);
|
||||
|
||||
tracing::info!("AUTH server shutting down, draining remaining connections...");
|
||||
while connections.next().await.is_some() {}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
struct NetLoop {
|
||||
login: ArcLoginProvider,
|
||||
stream: BufStream<TcpStream>,
|
||||
stop: watch::Receiver<bool>,
|
||||
state: State,
|
||||
read_buf: Vec<u8>,
|
||||
write_buf: BytesMut,
|
||||
}
|
||||
|
||||
impl NetLoop {
|
||||
fn new(stream: TcpStream, login: ArcLoginProvider, stop: watch::Receiver<bool>) -> Self {
|
||||
Self {
|
||||
login,
|
||||
stream: BufStream::new(stream),
|
||||
state: State::Init,
|
||||
stop,
|
||||
read_buf: Vec::new(),
|
||||
write_buf: BytesMut::new(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_error(self) {
|
||||
match self.run().await {
|
||||
Ok(()) => tracing::info!("Auth session succeeded"),
|
||||
Err(e) => tracing::error!(err=?e, "Auth session failed"),
|
||||
}
|
||||
}
|
||||
|
||||
async fn run(mut self) -> Result<()> {
|
||||
loop {
|
||||
tokio::select! {
|
||||
read_res = self.stream.read_until(b'\n', &mut self.read_buf) => {
|
||||
// Detect EOF / socket close
|
||||
let bread = read_res?;
|
||||
if bread == 0 {
|
||||
tracing::info!("Reading buffer empty, connection has been closed. Exiting AUTH session.");
|
||||
return Ok(())
|
||||
}
|
||||
|
||||
// Parse command
|
||||
let (_, cmd) = client_command(&self.read_buf).map_err(|_| anyhow!("Unable to parse command"))?;
|
||||
tracing::trace!(cmd=?cmd, "Received command");
|
||||
|
||||
// Make some progress in our local state
|
||||
self.state.progress(cmd, &self.login).await;
|
||||
if matches!(self.state, State::Error) {
|
||||
bail!("Internal state is in error, previous logs explain what went wrong");
|
||||
}
|
||||
|
||||
// Build response
|
||||
let srv_cmds = self.state.response();
|
||||
srv_cmds.iter().try_for_each(|r| {
|
||||
tracing::trace!(cmd=?r, "Sent command");
|
||||
r.encode(&mut self.write_buf)
|
||||
})?;
|
||||
|
||||
// Send responses if at least one command response has been generated
|
||||
if !srv_cmds.is_empty() {
|
||||
self.stream.write_all(&self.write_buf).await?;
|
||||
self.stream.flush().await?;
|
||||
}
|
||||
|
||||
// Reset buffers
|
||||
self.read_buf.clear();
|
||||
self.write_buf.clear();
|
||||
},
|
||||
_ = self.stop.changed() => {
|
||||
tracing::debug!("Server is stopping, quitting this runner");
|
||||
return Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------
|
||||
//
|
||||
// BUSINESS LOGIC
|
||||
//
|
||||
// -----------------------------------------------------------------
|
||||
use rand::prelude::*;
|
||||
|
||||
#[derive(Debug)]
|
||||
enum AuthRes {
|
||||
Success(String),
|
||||
Failed(Option<String>, Option<FailCode>),
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
enum State {
|
||||
Error,
|
||||
Init,
|
||||
HandshakePart(Version),
|
||||
HandshakeDone,
|
||||
AuthPlainProgress { id: u64 },
|
||||
AuthDone { id: u64, res: AuthRes },
|
||||
}
|
||||
|
||||
const SERVER_MAJOR: u64 = 1;
|
||||
const SERVER_MINOR: u64 = 2;
|
||||
const EMPTY_AUTHZ: &[u8] = &[];
|
||||
impl State {
|
||||
async fn try_auth_plain<'a>(&self, data: &'a [u8], login: &ArcLoginProvider) -> AuthRes {
|
||||
// Check that we can extract user's login+pass
|
||||
let (ubin, pbin) = match auth_plain(&data) {
|
||||
Ok(([], (authz, user, pass))) if authz == user || authz == EMPTY_AUTHZ => (user, pass),
|
||||
Ok(_) => {
|
||||
tracing::error!("Impersonating user is not supported");
|
||||
return AuthRes::Failed(None, None);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!(err=?e, "Could not parse the SASL PLAIN data chunk");
|
||||
return AuthRes::Failed(None, None);
|
||||
}
|
||||
};
|
||||
|
||||
// Try to convert it to UTF-8
|
||||
let (user, password) = match (std::str::from_utf8(ubin), std::str::from_utf8(pbin)) {
|
||||
(Ok(u), Ok(p)) => (u, p),
|
||||
_ => {
|
||||
tracing::error!("Username or password contain invalid UTF-8 characters");
|
||||
return AuthRes::Failed(None, None);
|
||||
}
|
||||
};
|
||||
|
||||
// Try to connect user
|
||||
match login.login(user, password).await {
|
||||
Ok(_) => AuthRes::Success(user.to_string()),
|
||||
Err(e) => {
|
||||
tracing::warn!(err=?e, "login failed");
|
||||
AuthRes::Failed(Some(user.to_string()), None)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn progress(&mut self, cmd: ClientCommand, login: &ArcLoginProvider) {
|
||||
let new_state = 'state: {
|
||||
match (std::mem::replace(self, State::Error), cmd) {
|
||||
(Self::Init, ClientCommand::Version(v)) => Self::HandshakePart(v),
|
||||
(Self::HandshakePart(version), ClientCommand::Cpid(_cpid)) => {
|
||||
if version.major != SERVER_MAJOR {
|
||||
tracing::error!(
|
||||
client_major = version.major,
|
||||
server_major = SERVER_MAJOR,
|
||||
"Unsupported client major version"
|
||||
);
|
||||
break 'state Self::Error;
|
||||
}
|
||||
|
||||
Self::HandshakeDone
|
||||
}
|
||||
(
|
||||
Self::HandshakeDone { .. },
|
||||
ClientCommand::Auth {
|
||||
id, mech, options, ..
|
||||
},
|
||||
)
|
||||
| (
|
||||
Self::AuthDone { .. },
|
||||
ClientCommand::Auth {
|
||||
id, mech, options, ..
|
||||
},
|
||||
) => {
|
||||
if mech != Mechanism::Plain {
|
||||
tracing::error!(mechanism=?mech, "Unsupported Authentication Mechanism");
|
||||
break 'state Self::AuthDone {
|
||||
id,
|
||||
res: AuthRes::Failed(None, None),
|
||||
};
|
||||
}
|
||||
|
||||
match options.last() {
|
||||
Some(AuthOption::Resp(data)) => Self::AuthDone {
|
||||
id,
|
||||
res: self.try_auth_plain(&data, login).await,
|
||||
},
|
||||
_ => Self::AuthPlainProgress { id },
|
||||
}
|
||||
}
|
||||
(Self::AuthPlainProgress { id }, ClientCommand::Cont { id: cid, data }) => {
|
||||
// Check that ID matches
|
||||
if cid != id {
|
||||
tracing::error!(
|
||||
auth_id = id,
|
||||
cont_id = cid,
|
||||
"CONT id does not match AUTH id"
|
||||
);
|
||||
break 'state Self::AuthDone {
|
||||
id,
|
||||
res: AuthRes::Failed(None, None),
|
||||
};
|
||||
}
|
||||
|
||||
Self::AuthDone {
|
||||
id,
|
||||
res: self.try_auth_plain(&data, login).await,
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
tracing::error!("This command is not valid in this context");
|
||||
Self::Error
|
||||
}
|
||||
}
|
||||
};
|
||||
tracing::debug!(state=?new_state, "Made progress");
|
||||
*self = new_state;
|
||||
}
|
||||
|
||||
fn response(&self) -> Vec<ServerCommand> {
|
||||
let mut srv_cmd: Vec<ServerCommand> = Vec::new();
|
||||
|
||||
match self {
|
||||
Self::HandshakeDone { .. } => {
|
||||
srv_cmd.push(ServerCommand::Version(Version {
|
||||
major: SERVER_MAJOR,
|
||||
minor: SERVER_MINOR,
|
||||
}));
|
||||
|
||||
srv_cmd.push(ServerCommand::Mech {
|
||||
kind: Mechanism::Plain,
|
||||
parameters: vec![MechanismParameters::PlainText],
|
||||
});
|
||||
|
||||
srv_cmd.push(ServerCommand::Spid(15u64));
|
||||
srv_cmd.push(ServerCommand::Cuid(19350u64));
|
||||
|
||||
let mut cookie = [0u8; 16];
|
||||
thread_rng().fill(&mut cookie);
|
||||
srv_cmd.push(ServerCommand::Cookie(cookie));
|
||||
|
||||
srv_cmd.push(ServerCommand::Done);
|
||||
}
|
||||
Self::AuthPlainProgress { id } => {
|
||||
srv_cmd.push(ServerCommand::Cont {
|
||||
id: *id,
|
||||
data: None,
|
||||
});
|
||||
}
|
||||
Self::AuthDone {
|
||||
id,
|
||||
res: AuthRes::Success(user),
|
||||
} => {
|
||||
srv_cmd.push(ServerCommand::Ok {
|
||||
id: *id,
|
||||
user_id: Some(user.to_string()),
|
||||
extra_parameters: vec![],
|
||||
});
|
||||
}
|
||||
Self::AuthDone {
|
||||
id,
|
||||
res: AuthRes::Failed(maybe_user, maybe_failcode),
|
||||
} => {
|
||||
srv_cmd.push(ServerCommand::Fail {
|
||||
id: *id,
|
||||
user_id: maybe_user.clone(),
|
||||
code: maybe_failcode.clone(),
|
||||
extra_parameters: vec![],
|
||||
});
|
||||
}
|
||||
_ => (),
|
||||
};
|
||||
|
||||
srv_cmd
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------
|
||||
//
|
||||
// DOVECOT AUTH TYPES
|
||||
//
|
||||
// -----------------------------------------------------------------
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
enum Mechanism {
|
||||
Plain,
|
||||
Login,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
enum AuthOption {
|
||||
/// Unique session ID. Mainly used for logging.
|
||||
Session(u64),
|
||||
/// Local IP connected to by the client. In standard string format, e.g. 127.0.0.1 or ::1.
|
||||
LocalIp(String),
|
||||
/// Remote client IP
|
||||
RemoteIp(String),
|
||||
/// Local port connected to by the client.
|
||||
LocalPort(u16),
|
||||
/// Remote client port
|
||||
RemotePort(u16),
|
||||
/// When Dovecot proxy is used, the real_rip/real_port are the proxy’s IP/port and real_lip/real_lport are the backend’s IP/port where the proxy was connected to.
|
||||
RealRemoteIp(String),
|
||||
RealLocalIp(String),
|
||||
RealLocalPort(u16),
|
||||
RealRemotePort(u16),
|
||||
/// TLS SNI name
|
||||
LocalName(String),
|
||||
/// Enable debugging for this lookup.
|
||||
Debug,
|
||||
/// List of fields that will become available via %{forward_*} variables. The list is double-tab-escaped, like: tab_escaped[tab_escaped(key=value)[<TAB>...]
|
||||
/// Note: we do not unescape the tabulation, and thus we don't parse the data
|
||||
ForwardViews(Vec<u8>),
|
||||
/// Remote user has secured transport to auth client (e.g. localhost, SSL, TLS).
|
||||
Secured(Option<String>),
|
||||
/// The value can be “insecure”, “trusted” or “TLS”.
|
||||
Transport(String),
|
||||
/// TLS cipher being used.
|
||||
TlsCipher(String),
|
||||
/// The number of bits in the TLS cipher.
|
||||
/// @FIXME: I don't know how if it's a string or an integer
|
||||
TlsCipherBits(String),
|
||||
/// TLS perfect forward secrecy algorithm (e.g. DH, ECDH)
|
||||
TlsPfs(String),
|
||||
/// TLS protocol name (e.g. SSLv3, TLSv1.2)
|
||||
TlsProtocol(String),
|
||||
/// Remote user has presented a valid SSL certificate.
|
||||
ValidClientCert(String),
|
||||
/// Ignore auth penalty tracking for this request
|
||||
NoPenalty,
|
||||
/// Unknown option sent by Postfix
|
||||
NoLogin,
|
||||
/// Username taken from client’s SSL certificate.
|
||||
CertUsername,
|
||||
/// IMAP ID string
|
||||
ClientId,
|
||||
/// An unknown key
|
||||
UnknownPair(String, Vec<u8>),
|
||||
UnknownBool(Vec<u8>),
|
||||
/// Initial response for authentication mechanism.
|
||||
/// NOTE: This must be the last parameter. Everything after it is ignored.
|
||||
/// This is to avoid accidental security holes if user-given data is directly put to base64 string without filtering out tabs.
|
||||
/// @FIXME: I don't understand this parameter
|
||||
Resp(Vec<u8>),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct Version {
|
||||
major: u64,
|
||||
minor: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
enum ClientCommand {
|
||||
/// Both client and server should check that they support the same major version number. If they don’t, the other side isn’t expected to be talking the same protocol and should be disconnected. Minor version can be ignored. This document specifies the version number 1.2.
|
||||
Version(Version),
|
||||
/// CPID finishes the handshake from client.
|
||||
Cpid(u64),
|
||||
Auth {
|
||||
/// ID is a connection-specific unique request identifier. It must be a 32bit number, so typically you’d just increment it by one.
|
||||
id: u64,
|
||||
/// A SASL mechanism (eg. LOGIN, PLAIN, etc.)
|
||||
/// See: https://doc.dovecot.org/configuration_manual/authentication/authentication_mechanisms/#authentication-authentication-mechanisms
|
||||
mech: Mechanism,
|
||||
/// Service is the service requesting authentication, eg. pop3, imap, smtp.
|
||||
service: String,
|
||||
/// All the optional parameters
|
||||
options: Vec<AuthOption>,
|
||||
},
|
||||
Cont {
|
||||
/// The <id> must match the <id> of the AUTH command.
|
||||
id: u64,
|
||||
/// Data that will be serialized to / deserialized from base64
|
||||
data: Vec<u8>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
enum MechanismParameters {
|
||||
/// Anonymous authentication
|
||||
Anonymous,
|
||||
/// Transfers plaintext passwords
|
||||
PlainText,
|
||||
/// Subject to passive (dictionary) attack
|
||||
Dictionary,
|
||||
/// Subject to active (non-dictionary) attack
|
||||
Active,
|
||||
/// Provides forward secrecy between sessions
|
||||
ForwardSecrecy,
|
||||
/// Provides mutual authentication
|
||||
MutualAuth,
|
||||
/// Don’t advertise this as available SASL mechanism (eg. APOP)
|
||||
Private,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
enum FailCode {
|
||||
/// This is a temporary internal failure, e.g. connection was lost to SQL database.
|
||||
TempFail,
|
||||
/// Authentication succeeded, but authorization failed (master user’s password was ok, but destination user was not ok).
|
||||
AuthzFail,
|
||||
/// User is disabled (password may or may not have been correct)
|
||||
UserDisabled,
|
||||
/// User’s password has expired.
|
||||
PassExpired,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
enum ServerCommand {
|
||||
/// Both client and server should check that they support the same major version number. If they don’t, the other side isn’t expected to be talking the same protocol and should be disconnected. Minor version can be ignored. This document specifies the version number 1.2.
|
||||
Version(Version),
|
||||
/// CPID and SPID specify client and server Process Identifiers (PIDs). They should be unique identifiers for the specific process. UNIX process IDs are good choices.
|
||||
/// SPID can be used by authentication client to tell master which server process handled the authentication.
|
||||
Spid(u64),
|
||||
/// CUID is a server process-specific unique connection identifier. It’s different each time a connection is established for the server.
|
||||
/// CUID is currently useful only for APOP authentication.
|
||||
Cuid(u64),
|
||||
Mech {
|
||||
kind: Mechanism,
|
||||
parameters: Vec<MechanismParameters>,
|
||||
},
|
||||
/// COOKIE returns connection-specific 128 bit cookie in hex. It must be given to REQUEST command. (Protocol v1.1+ / Dovecot v2.0+)
|
||||
Cookie([u8; 16]),
|
||||
/// DONE finishes the handshake from server.
|
||||
Done,
|
||||
|
||||
Fail {
|
||||
id: u64,
|
||||
user_id: Option<String>,
|
||||
code: Option<FailCode>,
|
||||
extra_parameters: Vec<Vec<u8>>,
|
||||
},
|
||||
Cont {
|
||||
id: u64,
|
||||
data: Option<Vec<u8>>,
|
||||
},
|
||||
/// FAIL and OK may contain multiple unspecified parameters which authentication client may handle specially.
|
||||
/// The only one specified here is user=<userid> parameter, which should always be sent if the userid is known.
|
||||
Ok {
|
||||
id: u64,
|
||||
user_id: Option<String>,
|
||||
extra_parameters: Vec<Vec<u8>>,
|
||||
},
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------
|
||||
//
|
||||
// DOVECOT AUTH DECODING
|
||||
//
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
use base64::Engine;
|
||||
use nom::{
|
||||
branch::alt,
|
||||
bytes::complete::{is_not, tag, tag_no_case, take, take_while, take_while1},
|
||||
character::complete::{tab, u16, u64},
|
||||
combinator::{map, opt, recognize, rest, value},
|
||||
error::{Error, ErrorKind},
|
||||
multi::{many1, separated_list0},
|
||||
sequence::{pair, preceded, tuple},
|
||||
IResult,
|
||||
};
|
||||
|
||||
fn version_command<'a>(input: &'a [u8]) -> IResult<&'a [u8], ClientCommand> {
|
||||
let mut parser = tuple((tag_no_case(b"VERSION"), tab, u64, tab, u64));
|
||||
|
||||
let (input, (_, _, major, _, minor)) = parser(input)?;
|
||||
Ok((input, ClientCommand::Version(Version { major, minor })))
|
||||
}
|
||||
|
||||
fn cpid_command<'a>(input: &'a [u8]) -> IResult<&'a [u8], ClientCommand> {
|
||||
preceded(
|
||||
pair(tag_no_case(b"CPID"), tab),
|
||||
map(u64, |v| ClientCommand::Cpid(v)),
|
||||
)(input)
|
||||
}
|
||||
|
||||
fn mechanism<'a>(input: &'a [u8]) -> IResult<&'a [u8], Mechanism> {
|
||||
alt((
|
||||
value(Mechanism::Plain, tag_no_case(b"PLAIN")),
|
||||
value(Mechanism::Login, tag_no_case(b"LOGIN")),
|
||||
))(input)
|
||||
}
|
||||
|
||||
fn is_not_tab_or_esc_or_lf(c: u8) -> bool {
|
||||
c != 0x09 && c != 0x01 && c != 0x0a // TAB or 0x01 or LF
|
||||
}
|
||||
|
||||
fn is_esc<'a>(input: &'a [u8]) -> IResult<&'a [u8], &[u8]> {
|
||||
preceded(tag(&[0x01]), take(1usize))(input)
|
||||
}
|
||||
|
||||
fn parameter<'a>(input: &'a [u8]) -> IResult<&'a [u8], &[u8]> {
|
||||
recognize(many1(alt((take_while1(is_not_tab_or_esc_or_lf), is_esc))))(input)
|
||||
}
|
||||
|
||||
fn parameter_str(input: &[u8]) -> IResult<&[u8], String> {
|
||||
let (input, buf) = parameter(input)?;
|
||||
|
||||
std::str::from_utf8(buf)
|
||||
.map(|v| (input, v.to_string()))
|
||||
.map_err(|_| nom::Err::Failure(Error::new(input, ErrorKind::TakeWhile1)))
|
||||
}
|
||||
|
||||
fn is_param_name_char(c: u8) -> bool {
|
||||
is_not_tab_or_esc_or_lf(c) && c != 0x3d // =
|
||||
}
|
||||
|
||||
fn parameter_name(input: &[u8]) -> IResult<&[u8], String> {
|
||||
let (input, buf) = take_while1(is_param_name_char)(input)?;
|
||||
|
||||
std::str::from_utf8(buf)
|
||||
.map(|v| (input, v.to_string()))
|
||||
.map_err(|_| nom::Err::Failure(Error::new(input, ErrorKind::TakeWhile1)))
|
||||
}
|
||||
|
||||
fn service<'a>(input: &'a [u8]) -> IResult<&'a [u8], String> {
|
||||
preceded(tag_no_case("service="), parameter_str)(input)
|
||||
}
|
||||
|
||||
fn auth_option<'a>(input: &'a [u8]) -> IResult<&'a [u8], AuthOption> {
|
||||
use AuthOption::*;
|
||||
alt((
|
||||
alt((
|
||||
value(Debug, tag_no_case(b"debug")),
|
||||
value(NoPenalty, tag_no_case(b"no-penalty")),
|
||||
value(ClientId, tag_no_case(b"client_id")),
|
||||
value(NoLogin, tag_no_case(b"nologin")),
|
||||
map(preceded(tag_no_case(b"session="), u64), |id| Session(id)),
|
||||
map(preceded(tag_no_case(b"lip="), parameter_str), |ip| {
|
||||
LocalIp(ip)
|
||||
}),
|
||||
map(preceded(tag_no_case(b"rip="), parameter_str), |ip| {
|
||||
RemoteIp(ip)
|
||||
}),
|
||||
map(preceded(tag_no_case(b"lport="), u16), |port| {
|
||||
LocalPort(port)
|
||||
}),
|
||||
map(preceded(tag_no_case(b"rport="), u16), |port| {
|
||||
RemotePort(port)
|
||||
}),
|
||||
map(preceded(tag_no_case(b"real_rip="), parameter_str), |ip| {
|
||||
RealRemoteIp(ip)
|
||||
}),
|
||||
map(preceded(tag_no_case(b"real_lip="), parameter_str), |ip| {
|
||||
RealLocalIp(ip)
|
||||
}),
|
||||
map(preceded(tag_no_case(b"real_lport="), u16), |port| {
|
||||
RealLocalPort(port)
|
||||
}),
|
||||
map(preceded(tag_no_case(b"real_rport="), u16), |port| {
|
||||
RealRemotePort(port)
|
||||
}),
|
||||
)),
|
||||
alt((
|
||||
map(
|
||||
preceded(tag_no_case(b"local_name="), parameter_str),
|
||||
|name| LocalName(name),
|
||||
),
|
||||
map(
|
||||
preceded(tag_no_case(b"forward_views="), parameter),
|
||||
|views| ForwardViews(views.into()),
|
||||
),
|
||||
map(preceded(tag_no_case(b"secured="), parameter_str), |info| {
|
||||
Secured(Some(info))
|
||||
}),
|
||||
value(Secured(None), tag_no_case(b"secured")),
|
||||
value(CertUsername, tag_no_case(b"cert_username")),
|
||||
map(preceded(tag_no_case(b"transport="), parameter_str), |ts| {
|
||||
Transport(ts)
|
||||
}),
|
||||
map(
|
||||
preceded(tag_no_case(b"tls_cipher="), parameter_str),
|
||||
|cipher| TlsCipher(cipher),
|
||||
),
|
||||
map(
|
||||
preceded(tag_no_case(b"tls_cipher_bits="), parameter_str),
|
||||
|bits| TlsCipherBits(bits),
|
||||
),
|
||||
map(preceded(tag_no_case(b"tls_pfs="), parameter_str), |pfs| {
|
||||
TlsPfs(pfs)
|
||||
}),
|
||||
map(
|
||||
preceded(tag_no_case(b"tls_protocol="), parameter_str),
|
||||
|proto| TlsProtocol(proto),
|
||||
),
|
||||
map(
|
||||
preceded(tag_no_case(b"valid-client-cert="), parameter_str),
|
||||
|cert| ValidClientCert(cert),
|
||||
),
|
||||
)),
|
||||
alt((
|
||||
map(preceded(tag_no_case(b"resp="), base64), |data| Resp(data)),
|
||||
map(
|
||||
tuple((parameter_name, tag(b"="), parameter)),
|
||||
|(n, _, v)| UnknownPair(n, v.into()),
|
||||
),
|
||||
map(parameter, |v| UnknownBool(v.into())),
|
||||
)),
|
||||
))(input)
|
||||
}
|
||||
|
||||
fn auth_command<'a>(input: &'a [u8]) -> IResult<&'a [u8], ClientCommand> {
|
||||
let mut parser = tuple((
|
||||
tag_no_case(b"AUTH"),
|
||||
tab,
|
||||
u64,
|
||||
tab,
|
||||
mechanism,
|
||||
tab,
|
||||
service,
|
||||
map(opt(preceded(tab, separated_list0(tab, auth_option))), |o| {
|
||||
o.unwrap_or(vec![])
|
||||
}),
|
||||
));
|
||||
let (input, (_, _, id, _, mech, _, service, options)) = parser(input)?;
|
||||
Ok((
|
||||
input,
|
||||
ClientCommand::Auth {
|
||||
id,
|
||||
mech,
|
||||
service,
|
||||
options,
|
||||
},
|
||||
))
|
||||
}
|
||||
|
||||
fn is_base64_core(c: u8) -> bool {
|
||||
c >= 0x30 && c <= 0x39 // 0-9
|
||||
|| c >= 0x41 && c <= 0x5a // A-Z
|
||||
|| c >= 0x61 && c <= 0x7a // a-z
|
||||
|| c == 0x2b // +
|
||||
|| c == 0x2f // /
|
||||
}
|
||||
|
||||
fn is_base64_pad(c: u8) -> bool {
|
||||
c == 0x3d // =
|
||||
}
|
||||
|
||||
fn base64(input: &[u8]) -> IResult<&[u8], Vec<u8>> {
|
||||
let (input, (b64, _)) = tuple((take_while1(is_base64_core), take_while(is_base64_pad)))(input)?;
|
||||
|
||||
let data = base64::engine::general_purpose::STANDARD_NO_PAD
|
||||
.decode(b64)
|
||||
.map_err(|_| nom::Err::Failure(Error::new(input, ErrorKind::TakeWhile1)))?;
|
||||
|
||||
Ok((input, data))
|
||||
}
|
||||
|
||||
/// @FIXME Dovecot does not say if base64 content must be padded or not
|
||||
fn cont_command<'a>(input: &'a [u8]) -> IResult<&'a [u8], ClientCommand> {
|
||||
let mut parser = tuple((tag_no_case(b"CONT"), tab, u64, tab, base64));
|
||||
|
||||
let (input, (_, _, id, _, data)) = parser(input)?;
|
||||
Ok((input, ClientCommand::Cont { id, data }))
|
||||
}
|
||||
|
||||
fn client_command<'a>(input: &'a [u8]) -> IResult<&'a [u8], ClientCommand> {
|
||||
alt((version_command, cpid_command, auth_command, cont_command))(input)
|
||||
}
|
||||
|
||||
/*
|
||||
fn server_command(buf: &u8) -> IResult<&u8, ServerCommand> {
|
||||
unimplemented!();
|
||||
}
|
||||
*/
|
||||
|
||||
// -----------------------------------------------------------------
|
||||
//
|
||||
// SASL DECODING
|
||||
//
|
||||
// -----------------------------------------------------------------
|
||||
|
||||
fn not_null(c: u8) -> bool {
|
||||
c != 0x0
|
||||
}
|
||||
|
||||
// impersonated user, login, password
|
||||
fn auth_plain<'a>(input: &'a [u8]) -> IResult<&'a [u8], (&'a [u8], &'a [u8], &'a [u8])> {
|
||||
map(
|
||||
tuple((
|
||||
take_while(not_null),
|
||||
take(1usize),
|
||||
take_while(not_null),
|
||||
take(1usize),
|
||||
rest,
|
||||
)),
|
||||
|(imp, _, user, _, pass)| (imp, user, pass),
|
||||
)(input)
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------
|
||||
//
|
||||
// DOVECOT AUTH ENCODING
|
||||
//
|
||||
// ------------------------------------------------------------------
|
||||
use tokio_util::bytes::{BufMut, BytesMut};
|
||||
trait Encode {
|
||||
fn encode(&self, out: &mut BytesMut) -> Result<()>;
|
||||
}
|
||||
|
||||
fn tab_enc(out: &mut BytesMut) {
|
||||
out.put(&[0x09][..])
|
||||
}
|
||||
|
||||
fn lf_enc(out: &mut BytesMut) {
|
||||
out.put(&[0x0A][..])
|
||||
}
|
||||
|
||||
impl Encode for Mechanism {
|
||||
fn encode(&self, out: &mut BytesMut) -> Result<()> {
|
||||
match self {
|
||||
Self::Plain => out.put(&b"PLAIN"[..]),
|
||||
Self::Login => out.put(&b"LOGIN"[..]),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Encode for MechanismParameters {
|
||||
fn encode(&self, out: &mut BytesMut) -> Result<()> {
|
||||
match self {
|
||||
Self::Anonymous => out.put(&b"anonymous"[..]),
|
||||
Self::PlainText => out.put(&b"plaintext"[..]),
|
||||
Self::Dictionary => out.put(&b"dictionary"[..]),
|
||||
Self::Active => out.put(&b"active"[..]),
|
||||
Self::ForwardSecrecy => out.put(&b"forward-secrecy"[..]),
|
||||
Self::MutualAuth => out.put(&b"mutual-auth"[..]),
|
||||
Self::Private => out.put(&b"private"[..]),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Encode for FailCode {
|
||||
fn encode(&self, out: &mut BytesMut) -> Result<()> {
|
||||
match self {
|
||||
Self::TempFail => out.put(&b"temp_fail"[..]),
|
||||
Self::AuthzFail => out.put(&b"authz_fail"[..]),
|
||||
Self::UserDisabled => out.put(&b"user_disabled"[..]),
|
||||
Self::PassExpired => out.put(&b"pass_expired"[..]),
|
||||
};
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Encode for ServerCommand {
|
||||
fn encode(&self, out: &mut BytesMut) -> Result<()> {
|
||||
match self {
|
||||
Self::Version(Version { major, minor }) => {
|
||||
out.put(&b"VERSION"[..]);
|
||||
tab_enc(out);
|
||||
out.put(major.to_string().as_bytes());
|
||||
tab_enc(out);
|
||||
out.put(minor.to_string().as_bytes());
|
||||
lf_enc(out);
|
||||
}
|
||||
Self::Spid(pid) => {
|
||||
out.put(&b"SPID"[..]);
|
||||
tab_enc(out);
|
||||
out.put(pid.to_string().as_bytes());
|
||||
lf_enc(out);
|
||||
}
|
||||
Self::Cuid(pid) => {
|
||||
out.put(&b"CUID"[..]);
|
||||
tab_enc(out);
|
||||
out.put(pid.to_string().as_bytes());
|
||||
lf_enc(out);
|
||||
}
|
||||
Self::Cookie(cval) => {
|
||||
out.put(&b"COOKIE"[..]);
|
||||
tab_enc(out);
|
||||
out.put(hex::encode(cval).as_bytes());
|
||||
lf_enc(out);
|
||||
}
|
||||
Self::Mech { kind, parameters } => {
|
||||
out.put(&b"MECH"[..]);
|
||||
tab_enc(out);
|
||||
kind.encode(out)?;
|
||||
for p in parameters.iter() {
|
||||
tab_enc(out);
|
||||
p.encode(out)?;
|
||||
}
|
||||
lf_enc(out);
|
||||
}
|
||||
Self::Done => {
|
||||
out.put(&b"DONE"[..]);
|
||||
lf_enc(out);
|
||||
}
|
||||
Self::Cont { id, data } => {
|
||||
out.put(&b"CONT"[..]);
|
||||
tab_enc(out);
|
||||
out.put(id.to_string().as_bytes());
|
||||
tab_enc(out);
|
||||
if let Some(rdata) = data {
|
||||
let b64 = base64::engine::general_purpose::STANDARD.encode(rdata);
|
||||
out.put(b64.as_bytes());
|
||||
}
|
||||
lf_enc(out);
|
||||
}
|
||||
Self::Ok {
|
||||
id,
|
||||
user_id,
|
||||
extra_parameters,
|
||||
} => {
|
||||
out.put(&b"OK"[..]);
|
||||
tab_enc(out);
|
||||
out.put(id.to_string().as_bytes());
|
||||
if let Some(user) = user_id {
|
||||
tab_enc(out);
|
||||
out.put(&b"user="[..]);
|
||||
out.put(user.as_bytes());
|
||||
}
|
||||
for p in extra_parameters.iter() {
|
||||
tab_enc(out);
|
||||
out.put(&p[..]);
|
||||
}
|
||||
lf_enc(out);
|
||||
}
|
||||
Self::Fail {
|
||||
id,
|
||||
user_id,
|
||||
code,
|
||||
extra_parameters,
|
||||
} => {
|
||||
out.put(&b"FAIL"[..]);
|
||||
tab_enc(out);
|
||||
out.put(id.to_string().as_bytes());
|
||||
if let Some(user) = user_id {
|
||||
tab_enc(out);
|
||||
out.put(&b"user="[..]);
|
||||
out.put(user.as_bytes());
|
||||
}
|
||||
if let Some(code_val) = code {
|
||||
tab_enc(out);
|
||||
out.put(&b"code="[..]);
|
||||
code_val.encode(out)?;
|
||||
}
|
||||
for p in extra_parameters.iter() {
|
||||
tab_enc(out);
|
||||
out.put(&p[..]);
|
||||
}
|
||||
lf_enc(out);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
386
src/bayou.rs
386
src/bayou.rs
|
|
@ -1,23 +1,27 @@
|
|||
use std::sync::{Arc, Weak};
|
||||
use std::str::FromStr;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use anyhow::{anyhow, bail, Result};
|
||||
use log::error;
|
||||
use log::debug;
|
||||
use rand::prelude::*;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::sync::{watch, Notify};
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
use k2v_client::{BatchDeleteOp, BatchReadOp, Filter, K2vClient, K2vValue};
|
||||
use rusoto_s3::{
|
||||
DeleteObjectRequest, GetObjectRequest, ListObjectsV2Request, PutObjectRequest, S3Client, S3,
|
||||
};
|
||||
|
||||
use crate::cryptoblob::*;
|
||||
use crate::login::Credentials;
|
||||
use crate::storage;
|
||||
use crate::timestamp::*;
|
||||
use crate::time::now_msec;
|
||||
|
||||
const KEEP_STATE_EVERY: usize = 64;
|
||||
const SAVE_STATE_EVERY: usize = 64;
|
||||
|
||||
// Checkpointing interval constants: a checkpoint is not made earlier
|
||||
// than CHECKPOINT_INTERVAL time after the last one, and is not made
|
||||
// if there are less than CHECKPOINT_MIN_OPS new operations since last one.
|
||||
const CHECKPOINT_INTERVAL: Duration = Duration::from_secs(6 * 3600);
|
||||
const CHECKPOINT_INTERVAL: Duration = Duration::from_secs(3600);
|
||||
const CHECKPOINT_MIN_OPS: usize = 16;
|
||||
// HYPOTHESIS: processes are able to communicate in a synchronous
|
||||
// fashion in times that are small compared to CHECKPOINT_INTERVAL.
|
||||
|
|
@ -30,8 +34,6 @@ const CHECKPOINT_MIN_OPS: usize = 16;
|
|||
// between processes doing .checkpoint() and those doing .sync()
|
||||
const CHECKPOINTS_TO_KEEP: usize = 3;
|
||||
|
||||
const WATCH_SK: &str = "watch";
|
||||
|
||||
pub trait BayouState:
|
||||
Default + Clone + Serialize + for<'de> Deserialize<'de> + Send + Sync + 'static
|
||||
{
|
||||
|
|
@ -41,64 +43,60 @@ pub trait BayouState:
|
|||
}
|
||||
|
||||
pub struct Bayou<S: BayouState> {
|
||||
bucket: String,
|
||||
path: String,
|
||||
key: Key,
|
||||
|
||||
storage: storage::Store,
|
||||
k2v: K2vClient,
|
||||
s3: S3Client,
|
||||
|
||||
checkpoint: (Timestamp, S),
|
||||
history: Vec<(Timestamp, S::Op, Option<S>)>,
|
||||
|
||||
last_sync: Option<Instant>,
|
||||
last_try_checkpoint: Option<Instant>,
|
||||
|
||||
watch: Arc<K2vWatch>,
|
||||
last_sync_watch_ct: storage::RowRef,
|
||||
}
|
||||
|
||||
impl<S: BayouState> Bayou<S> {
|
||||
pub async fn new(creds: &Credentials, path: String) -> Result<Self> {
|
||||
let storage = creds.storage.build().await?;
|
||||
|
||||
//let target = k2v_client.row(&path, WATCH_SK);
|
||||
let target = storage::RowRef::new(&path, WATCH_SK);
|
||||
let watch = K2vWatch::new(creds, target.clone()).await?;
|
||||
pub fn new(creds: &Credentials, path: String) -> Result<Self> {
|
||||
let k2v_client = creds.k2v_client()?;
|
||||
let s3_client = creds.s3_client()?;
|
||||
|
||||
Ok(Self {
|
||||
bucket: creds.bucket().to_string(),
|
||||
path,
|
||||
storage,
|
||||
key: creds.keys.master.clone(),
|
||||
k2v: k2v_client,
|
||||
s3: s3_client,
|
||||
checkpoint: (Timestamp::zero(), S::default()),
|
||||
history: vec![],
|
||||
last_sync: None,
|
||||
last_try_checkpoint: None,
|
||||
watch,
|
||||
last_sync_watch_ct: target,
|
||||
})
|
||||
}
|
||||
|
||||
/// Re-reads the state from persistent storage backend
|
||||
pub async fn sync(&mut self) -> Result<()> {
|
||||
let new_last_sync = Some(Instant::now());
|
||||
let new_last_sync_watch_ct = self.watch.rx.borrow().clone();
|
||||
|
||||
// 1. List checkpoints
|
||||
let checkpoints = self.list_checkpoints().await?;
|
||||
tracing::debug!("(sync) listed checkpoints: {:?}", checkpoints);
|
||||
debug!("(sync) listed checkpoints: {:?}", checkpoints);
|
||||
|
||||
// 2. Load last checkpoint if different from currently used one
|
||||
let checkpoint = if let Some((ts, key)) = checkpoints.last() {
|
||||
if *ts == self.checkpoint.0 {
|
||||
(*ts, None)
|
||||
} else {
|
||||
tracing::debug!("(sync) loading checkpoint: {}", key);
|
||||
debug!("(sync) loading checkpoint: {}", key);
|
||||
|
||||
let buf = self
|
||||
.storage
|
||||
.blob_fetch(&storage::BlobRef(key.to_string()))
|
||||
.await?
|
||||
.value;
|
||||
tracing::debug!("(sync) checkpoint body length: {}", buf.len());
|
||||
let mut gor = GetObjectRequest::default();
|
||||
gor.bucket = self.bucket.clone();
|
||||
gor.key = key.to_string();
|
||||
let obj_res = self.s3.get_object(gor).await?;
|
||||
|
||||
let obj_body = obj_res.body.ok_or(anyhow!("Missing object body"))?;
|
||||
let mut buf = Vec::with_capacity(obj_res.content_length.unwrap_or(128) as usize);
|
||||
obj_body.into_async_read().read_to_end(&mut buf).await?;
|
||||
|
||||
debug!("(sync) checkpoint body length: {}", buf.len());
|
||||
|
||||
let ck = open_deserialize::<S>(&buf, &self.key)?;
|
||||
(*ts, Some(ck))
|
||||
|
|
@ -108,11 +106,11 @@ impl<S: BayouState> Bayou<S> {
|
|||
};
|
||||
|
||||
if self.checkpoint.0 > checkpoint.0 {
|
||||
bail!("Loaded checkpoint is more recent than stored one");
|
||||
bail!("Existing checkpoint is more recent than stored one");
|
||||
}
|
||||
|
||||
if let Some(ck) = checkpoint.1 {
|
||||
tracing::debug!(
|
||||
debug!(
|
||||
"(sync) updating checkpoint to loaded state at {:?}",
|
||||
checkpoint.0
|
||||
);
|
||||
|
|
@ -127,41 +125,49 @@ impl<S: BayouState> Bayou<S> {
|
|||
|
||||
// 3. List all operations starting from checkpoint
|
||||
let ts_ser = self.checkpoint.0.to_string();
|
||||
tracing::debug!("(sync) looking up operations starting at {}", ts_ser);
|
||||
debug!("(sync) looking up operations starting at {}", ts_ser);
|
||||
let ops_map = self
|
||||
.storage
|
||||
.row_fetch(&storage::Selector::Range {
|
||||
shard: &self.path,
|
||||
sort_begin: &ts_ser,
|
||||
sort_end: WATCH_SK,
|
||||
})
|
||||
.await?;
|
||||
.k2v
|
||||
.read_batch(&[BatchReadOp {
|
||||
partition_key: &self.path,
|
||||
filter: Filter {
|
||||
start: Some(&ts_ser),
|
||||
end: None,
|
||||
prefix: None,
|
||||
limit: None,
|
||||
reverse: false,
|
||||
},
|
||||
single_item: false,
|
||||
conflicts_only: false,
|
||||
tombstones: false,
|
||||
}])
|
||||
.await?
|
||||
.into_iter()
|
||||
.next()
|
||||
.ok_or(anyhow!("Missing K2V result"))?
|
||||
.items;
|
||||
|
||||
let mut ops = vec![];
|
||||
for row_value in ops_map {
|
||||
let row = row_value.row_ref;
|
||||
let sort_key = row.uid.sort;
|
||||
let ts = sort_key
|
||||
for (tsstr, val) in ops_map {
|
||||
let ts = tsstr
|
||||
.parse::<Timestamp>()
|
||||
.map_err(|_| anyhow!("Invalid operation timestamp: {}", sort_key))?;
|
||||
|
||||
let val = row_value.value;
|
||||
if val.len() != 1 {
|
||||
bail!("Invalid operation, has {} values", val.len());
|
||||
.map_err(|_| anyhow!("Invalid operation timestamp: {}", tsstr))?;
|
||||
if val.value.len() != 1 {
|
||||
bail!("Invalid operation, has {} values", val.value.len());
|
||||
}
|
||||
match &val[0] {
|
||||
storage::Alternative::Value(v) => {
|
||||
let op = open_deserialize::<S::Op>(v, &self.key)?;
|
||||
tracing::trace!("(sync) operation {}: {:?}", sort_key, op);
|
||||
match &val.value[0] {
|
||||
K2vValue::Value(v) => {
|
||||
let op = open_deserialize::<S::Op>(&v, &self.key)?;
|
||||
debug!("(sync) operation {}: {} {:?}", tsstr, base64::encode(v), op);
|
||||
ops.push((ts, op));
|
||||
}
|
||||
storage::Alternative::Tombstone => {
|
||||
continue;
|
||||
K2vValue::Tombstone => {
|
||||
unreachable!();
|
||||
}
|
||||
}
|
||||
}
|
||||
ops.sort_by_key(|(ts, _)| *ts);
|
||||
tracing::debug!("(sync) {} operations", ops.len());
|
||||
debug!("(sync) {} operations", ops.len());
|
||||
|
||||
if ops.len() < self.history.len() {
|
||||
bail!("Some operations have disappeared from storage!");
|
||||
|
|
@ -180,9 +186,12 @@ impl<S: BayouState> Bayou<S> {
|
|||
let i0 = self
|
||||
.history
|
||||
.iter()
|
||||
.enumerate()
|
||||
.zip(ops.iter())
|
||||
.take_while(|((ts1, _, _), (ts2, _))| ts1 == ts2)
|
||||
.count();
|
||||
.skip_while(|((_, (ts1, _, _)), (ts2, _))| ts1 == ts2)
|
||||
.map(|((i, _), _)| i)
|
||||
.next()
|
||||
.unwrap_or(self.history.len());
|
||||
|
||||
if ops.len() > i0 {
|
||||
// Remove operations from first position where histories differ
|
||||
|
|
@ -206,7 +215,7 @@ impl<S: BayouState> Bayou<S> {
|
|||
// Now, apply all operations retrieved from storage after the common part
|
||||
for (ts, op) in ops.drain(i0..) {
|
||||
state = state.apply(&op);
|
||||
if (self.history.len() + 1) % KEEP_STATE_EVERY == 0 {
|
||||
if (self.history.len() + 1) % SAVE_STATE_EVERY == 0 {
|
||||
self.history.push((ts, op, Some(state.clone())));
|
||||
} else {
|
||||
self.history.push((ts, op, None));
|
||||
|
|
@ -217,37 +226,23 @@ impl<S: BayouState> Bayou<S> {
|
|||
self.history.last_mut().unwrap().2 = Some(state);
|
||||
}
|
||||
|
||||
// Save info that sync has been done
|
||||
self.last_sync = new_last_sync;
|
||||
self.last_sync_watch_ct = new_last_sync_watch_ct;
|
||||
self.last_sync = Some(Instant::now());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Does a sync() if either of the two conditions is met:
|
||||
/// - last sync was more than CHECKPOINT_INTERVAL/5 ago
|
||||
/// - a change was detected
|
||||
pub async fn opportunistic_sync(&mut self) -> Result<()> {
|
||||
let too_old = match self.last_sync {
|
||||
Some(t) => Instant::now() > t + (CHECKPOINT_INTERVAL / 5),
|
||||
_ => true,
|
||||
};
|
||||
let changed = self.last_sync_watch_ct != *self.watch.rx.borrow();
|
||||
if too_old || changed {
|
||||
self.sync().await?;
|
||||
async fn check_recent_sync(&mut self) -> Result<()> {
|
||||
match self.last_sync {
|
||||
Some(t) if (Instant::now() - t) < CHECKPOINT_INTERVAL / 10 => Ok(()),
|
||||
_ => self.sync().await,
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn notifier(&self) -> std::sync::Weak<Notify> {
|
||||
Arc::downgrade(&self.watch.learnt_remote_update)
|
||||
}
|
||||
|
||||
/// Applies a new operation on the state. Once this function returns,
|
||||
/// the operation has been safely persisted to storage backend.
|
||||
/// Make sure to call `.opportunistic_sync()` before doing this,
|
||||
/// and even before calculating the `op` argument given here.
|
||||
/// the option has been safely persisted to storage backend
|
||||
pub async fn push(&mut self, op: S::Op) -> Result<()> {
|
||||
tracing::debug!("(push) add operation: {:?}", op);
|
||||
self.check_recent_sync().await?;
|
||||
|
||||
debug!("(push) add operation: {:?}", op);
|
||||
|
||||
let ts = Timestamp::after(
|
||||
self.history
|
||||
|
|
@ -255,20 +250,21 @@ impl<S: BayouState> Bayou<S> {
|
|||
.map(|(ts, _, _)| ts)
|
||||
.unwrap_or(&self.checkpoint.0),
|
||||
);
|
||||
|
||||
let row_val = storage::RowVal::new(
|
||||
storage::RowRef::new(&self.path, &ts.to_string()),
|
||||
seal_serialize(&op, &self.key)?,
|
||||
);
|
||||
self.storage.row_insert(vec![row_val]).await?;
|
||||
self.watch.propagate_local_update.notify_one();
|
||||
self.k2v
|
||||
.insert_item(
|
||||
&self.path,
|
||||
&ts.to_string(),
|
||||
seal_serialize(&op, &self.key)?,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let new_state = self.state().apply(&op);
|
||||
self.history.push((ts, op, Some(new_state)));
|
||||
|
||||
// Clear previously saved state in history if not required
|
||||
let hlen = self.history.len();
|
||||
if hlen >= 2 && (hlen - 1) % KEEP_STATE_EVERY != 0 {
|
||||
if hlen >= 2 && (hlen - 1) % SAVE_STATE_EVERY != 0 {
|
||||
self.history[hlen - 2].2 = None;
|
||||
}
|
||||
|
||||
|
|
@ -280,7 +276,7 @@ impl<S: BayouState> Bayou<S> {
|
|||
/// Save a new checkpoint if previous checkpoint is too old
|
||||
pub async fn checkpoint(&mut self) -> Result<()> {
|
||||
match self.last_try_checkpoint {
|
||||
Some(ts) if Instant::now() - ts < CHECKPOINT_INTERVAL / 5 => Ok(()),
|
||||
Some(ts) if Instant::now() - ts < CHECKPOINT_INTERVAL / 10 => Ok(()),
|
||||
_ => {
|
||||
let res = self.checkpoint_internal().await;
|
||||
if res.is_ok() {
|
||||
|
|
@ -292,7 +288,7 @@ impl<S: BayouState> Bayou<S> {
|
|||
}
|
||||
|
||||
async fn checkpoint_internal(&mut self) -> Result<()> {
|
||||
self.sync().await?;
|
||||
self.check_recent_sync().await?;
|
||||
|
||||
// Check what would be the possible time for a checkpoint in the history we have
|
||||
let now = now_msec() as i128;
|
||||
|
|
@ -309,18 +305,18 @@ impl<S: BayouState> Bayou<S> {
|
|||
{
|
||||
Some(i) => i,
|
||||
None => {
|
||||
tracing::debug!("(cp) Oldest operation is too recent to trigger checkpoint");
|
||||
debug!("(cp) Oldest operation is too recent to trigger checkpoint");
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
|
||||
if i_cp < CHECKPOINT_MIN_OPS {
|
||||
tracing::debug!("(cp) Not enough old operations to trigger checkpoint");
|
||||
debug!("(cp) Not enough old operations to trigger checkpoint");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let ts_cp = self.history[i_cp].0;
|
||||
tracing::debug!(
|
||||
debug!(
|
||||
"(cp) we could checkpoint at time {} (index {} in history)",
|
||||
ts_cp.to_string(),
|
||||
i_cp
|
||||
|
|
@ -328,13 +324,13 @@ impl<S: BayouState> Bayou<S> {
|
|||
|
||||
// Check existing checkpoints: if last one is too recent, don't checkpoint again.
|
||||
let existing_checkpoints = self.list_checkpoints().await?;
|
||||
tracing::debug!("(cp) listed checkpoints: {:?}", existing_checkpoints);
|
||||
debug!("(cp) listed checkpoints: {:?}", existing_checkpoints);
|
||||
|
||||
if let Some(last_cp) = existing_checkpoints.last() {
|
||||
if (ts_cp.msec as i128 - last_cp.0.msec as i128)
|
||||
< CHECKPOINT_INTERVAL.as_millis() as i128
|
||||
{
|
||||
tracing::debug!(
|
||||
debug!(
|
||||
"(cp) last checkpoint is too recent: {}, not checkpointing",
|
||||
last_cp.0.to_string()
|
||||
);
|
||||
|
|
@ -342,7 +338,7 @@ impl<S: BayouState> Bayou<S> {
|
|||
}
|
||||
}
|
||||
|
||||
tracing::debug!("(cp) saving checkpoint at {}", ts_cp.to_string());
|
||||
debug!("(cp) saving checkpoint at {}", ts_cp.to_string());
|
||||
|
||||
// Calculate state at time of checkpoint
|
||||
let mut last_known_state = (0, &self.checkpoint.1);
|
||||
|
|
@ -358,13 +354,13 @@ impl<S: BayouState> Bayou<S> {
|
|||
|
||||
// Serialize and save checkpoint
|
||||
let cryptoblob = seal_serialize(&state_cp, &self.key)?;
|
||||
tracing::debug!("(cp) checkpoint body length: {}", cryptoblob.len());
|
||||
debug!("(cp) checkpoint body length: {}", cryptoblob.len());
|
||||
|
||||
let blob_val = storage::BlobVal::new(
|
||||
storage::BlobRef(format!("{}/checkpoint/{}", self.path, ts_cp.to_string())),
|
||||
cryptoblob.into(),
|
||||
);
|
||||
self.storage.blob_insert(blob_val).await?;
|
||||
let mut por = PutObjectRequest::default();
|
||||
por.bucket = self.bucket.clone();
|
||||
por.key = format!("{}/checkpoint/{}", self.path, ts_cp.to_string());
|
||||
por.body = Some(cryptoblob.into());
|
||||
self.s3.put_object(por).await?;
|
||||
|
||||
// Drop old checkpoints (but keep at least CHECKPOINTS_TO_KEEP of them)
|
||||
let ecp_len = existing_checkpoints.len();
|
||||
|
|
@ -373,21 +369,24 @@ impl<S: BayouState> Bayou<S> {
|
|||
|
||||
// Delete blobs
|
||||
for (_ts, key) in existing_checkpoints[..last_to_keep].iter() {
|
||||
tracing::debug!("(cp) drop old checkpoint {}", key);
|
||||
self.storage
|
||||
.blob_rm(&storage::BlobRef(key.to_string()))
|
||||
.await?;
|
||||
debug!("(cp) drop old checkpoint {}", key);
|
||||
let mut dor = DeleteObjectRequest::default();
|
||||
dor.bucket = self.bucket.clone();
|
||||
dor.key = key.to_string();
|
||||
self.s3.delete_object(dor).await?;
|
||||
}
|
||||
|
||||
// Delete corresponding range of operations
|
||||
let ts_ser = existing_checkpoints[last_to_keep].0.to_string();
|
||||
self.storage
|
||||
.row_rm(&storage::Selector::Range {
|
||||
shard: &self.path,
|
||||
sort_begin: "",
|
||||
sort_end: &ts_ser,
|
||||
})
|
||||
.await?
|
||||
self.k2v
|
||||
.delete_batch(&[BatchDeleteOp {
|
||||
partition_key: &self.path,
|
||||
prefix: None,
|
||||
start: None,
|
||||
end: Some(&ts_ser),
|
||||
single_item: false,
|
||||
}])
|
||||
.await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
|
@ -406,14 +405,20 @@ impl<S: BayouState> Bayou<S> {
|
|||
async fn list_checkpoints(&self) -> Result<Vec<(Timestamp, String)>> {
|
||||
let prefix = format!("{}/checkpoint/", self.path);
|
||||
|
||||
let checkpoints_res = self.storage.blob_list(&prefix).await?;
|
||||
let mut lor = ListObjectsV2Request::default();
|
||||
lor.bucket = self.bucket.clone();
|
||||
lor.max_keys = Some(1000);
|
||||
lor.prefix = Some(prefix.clone());
|
||||
|
||||
let checkpoints_res = self.s3.list_objects_v2(lor).await?;
|
||||
|
||||
let mut checkpoints = vec![];
|
||||
for object in checkpoints_res {
|
||||
let key = object.0;
|
||||
if let Some(ckid) = key.strip_prefix(&prefix) {
|
||||
if let Ok(ts) = ckid.parse::<Timestamp>() {
|
||||
checkpoints.push((ts, key.into()));
|
||||
for object in checkpoints_res.contents.unwrap_or_default() {
|
||||
if let Some(key) = object.key {
|
||||
if let Some(ckid) = key.strip_prefix(&prefix) {
|
||||
if let Ok(ts) = ckid.parse::<Timestamp>() {
|
||||
checkpoints.push((ts, key));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -422,93 +427,54 @@ impl<S: BayouState> Bayou<S> {
|
|||
}
|
||||
}
|
||||
|
||||
// ---- Bayou watch in K2V ----
|
||||
|
||||
struct K2vWatch {
|
||||
target: storage::RowRef,
|
||||
rx: watch::Receiver<storage::RowRef>,
|
||||
propagate_local_update: Notify,
|
||||
learnt_remote_update: Arc<Notify>,
|
||||
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)]
|
||||
pub struct Timestamp {
|
||||
pub msec: u64,
|
||||
pub rand: u64,
|
||||
}
|
||||
|
||||
impl K2vWatch {
|
||||
/// Creates a new watch and launches subordinate threads.
|
||||
/// These threads hold Weak pointers to the struct;
|
||||
/// they exit when the Arc is dropped.
|
||||
async fn new(creds: &Credentials, target: storage::RowRef) -> Result<Arc<Self>> {
|
||||
let storage = creds.storage.build().await?;
|
||||
|
||||
let (tx, rx) = watch::channel::<storage::RowRef>(target.clone());
|
||||
let propagate_local_update = Notify::new();
|
||||
let learnt_remote_update = Arc::new(Notify::new());
|
||||
|
||||
let watch = Arc::new(K2vWatch {
|
||||
target,
|
||||
rx,
|
||||
propagate_local_update,
|
||||
learnt_remote_update,
|
||||
});
|
||||
|
||||
tokio::spawn(Self::background_task(Arc::downgrade(&watch), storage, tx));
|
||||
|
||||
Ok(watch)
|
||||
}
|
||||
|
||||
async fn background_task(
|
||||
self_weak: Weak<Self>,
|
||||
storage: storage::Store,
|
||||
tx: watch::Sender<storage::RowRef>,
|
||||
) {
|
||||
let (mut row, remote_update) = match Weak::upgrade(&self_weak) {
|
||||
Some(this) => (this.target.clone(), this.learnt_remote_update.clone()),
|
||||
None => return,
|
||||
};
|
||||
|
||||
while let Some(this) = Weak::upgrade(&self_weak) {
|
||||
tracing::debug!(
|
||||
"bayou k2v watch bg loop iter ({}, {})",
|
||||
this.target.uid.shard,
|
||||
this.target.uid.sort
|
||||
);
|
||||
tokio::select!(
|
||||
// Needed to exit: will force a loop iteration every minutes,
|
||||
// that will stop the loop if other Arc references have been dropped
|
||||
// and free resources. Otherwise we would be blocked waiting forever...
|
||||
_ = tokio::time::sleep(Duration::from_secs(60)) => continue,
|
||||
|
||||
// Watch if another instance has modified the log
|
||||
update = storage.row_poll(&row) => {
|
||||
match update {
|
||||
Err(e) => {
|
||||
error!("Error in bayou k2v wait value changed: {}", e);
|
||||
tokio::time::sleep(Duration::from_secs(30)).await;
|
||||
}
|
||||
Ok(new_value) => {
|
||||
row = new_value.row_ref;
|
||||
if let Err(e) = tx.send(row.clone()) {
|
||||
tracing::warn!(err=?e, "(watch) can't record the new log ref");
|
||||
break;
|
||||
}
|
||||
tracing::debug!(row=?row, "(watch) learnt remote update");
|
||||
this.learnt_remote_update.notify_waiters();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// It appears we have modified the log, informing other people
|
||||
_ = this.propagate_local_update.notified() => {
|
||||
let rand = u128::to_be_bytes(thread_rng().gen()).to_vec();
|
||||
let row_val = storage::RowVal::new(row.clone(), rand);
|
||||
if let Err(e) = storage.row_insert(vec![row_val]).await
|
||||
{
|
||||
tracing::error!("Error in bayou k2v watch updater loop: {}", e);
|
||||
tokio::time::sleep(Duration::from_secs(30)).await;
|
||||
}
|
||||
}
|
||||
);
|
||||
impl Timestamp {
|
||||
pub fn now() -> Self {
|
||||
let mut rng = thread_rng();
|
||||
Self {
|
||||
msec: now_msec(),
|
||||
rand: rng.gen::<u64>(),
|
||||
}
|
||||
// unblock listeners
|
||||
remote_update.notify_waiters();
|
||||
tracing::info!("bayou k2v watch bg loop exiting");
|
||||
}
|
||||
|
||||
pub fn after(other: &Self) -> Self {
|
||||
let mut rng = thread_rng();
|
||||
Self {
|
||||
msec: std::cmp::max(now_msec(), other.msec + 1),
|
||||
rand: rng.gen::<u64>(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn zero() -> Self {
|
||||
Self { msec: 0, rand: 0 }
|
||||
}
|
||||
}
|
||||
|
||||
impl ToString for Timestamp {
|
||||
fn to_string(&self) -> String {
|
||||
let mut bytes = [0u8; 16];
|
||||
bytes[0..8].copy_from_slice(&u64::to_be_bytes(self.msec));
|
||||
bytes[8..16].copy_from_slice(&u64::to_be_bytes(self.rand));
|
||||
hex::encode(&bytes)
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for Timestamp {
|
||||
type Err = &'static str;
|
||||
|
||||
fn from_str(s: &str) -> Result<Timestamp, &'static str> {
|
||||
let bytes = hex::decode(s).map_err(|_| "invalid hex")?;
|
||||
if bytes.len() != 16 {
|
||||
return Err("bad length");
|
||||
}
|
||||
Ok(Self {
|
||||
msec: u64::from_be_bytes(bytes[0..8].try_into().unwrap()),
|
||||
rand: u64::from_be_bytes(bytes[8..16].try_into().unwrap()),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
191
src/config.rs
191
src/config.rs
|
|
@ -1,5 +1,5 @@
|
|||
use std::collections::HashMap;
|
||||
use std::io::{Read, Write};
|
||||
use std::io::Read;
|
||||
use std::net::SocketAddr;
|
||||
use std::path::PathBuf;
|
||||
|
||||
|
|
@ -7,35 +7,63 @@ use anyhow::Result;
|
|||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
pub struct CompanionConfig {
|
||||
pub pid: Option<PathBuf>,
|
||||
pub imap: ImapUnsecureConfig,
|
||||
pub struct Config {
|
||||
pub s3_endpoint: String,
|
||||
pub k2v_endpoint: String,
|
||||
pub aws_region: String,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub users: LoginStaticConfig,
|
||||
}
|
||||
pub login_static: Option<LoginStaticConfig>,
|
||||
pub login_ldap: Option<LoginLdapConfig>,
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
pub struct ProviderConfig {
|
||||
pub pid: Option<PathBuf>,
|
||||
pub imap: Option<ImapConfig>,
|
||||
pub imap_unsecure: Option<ImapUnsecureConfig>,
|
||||
pub lmtp: Option<LmtpConfig>,
|
||||
pub auth: Option<AuthConfig>,
|
||||
pub users: UserManagement,
|
||||
pub imap: Option<ImapConfig>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
#[serde(tag = "user_driver")]
|
||||
pub enum UserManagement {
|
||||
Demo,
|
||||
Static(LoginStaticConfig),
|
||||
Ldap(LoginLdapConfig),
|
||||
pub struct LoginStaticConfig {
|
||||
pub default_bucket: Option<String>,
|
||||
pub users: HashMap<String, LoginStaticUser>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
pub struct AuthConfig {
|
||||
pub bind_addr: SocketAddr,
|
||||
pub struct LoginStaticUser {
|
||||
#[serde(default)]
|
||||
pub email_addresses: Vec<String>,
|
||||
pub password: String,
|
||||
|
||||
pub aws_access_key_id: String,
|
||||
pub aws_secret_access_key: String,
|
||||
pub bucket: Option<String>,
|
||||
|
||||
pub user_secret: String,
|
||||
#[serde(default)]
|
||||
pub alternate_user_secrets: Vec<String>,
|
||||
|
||||
pub master_key: Option<String>,
|
||||
pub secret_key: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
pub struct LoginLdapConfig {
|
||||
pub ldap_server: String,
|
||||
|
||||
#[serde(default)]
|
||||
pub pre_bind_on_login: bool,
|
||||
pub bind_dn: Option<String>,
|
||||
pub bind_password: Option<String>,
|
||||
|
||||
pub search_base: String,
|
||||
pub username_attr: String,
|
||||
#[serde(default = "default_mail_attr")]
|
||||
pub mail_attr: String,
|
||||
|
||||
pub aws_access_key_id_attr: String,
|
||||
pub aws_secret_access_key_attr: String,
|
||||
pub user_secret_attr: String,
|
||||
pub alternate_user_secrets_attr: Option<String>,
|
||||
|
||||
pub bucket: Option<String>,
|
||||
pub bucket_attr: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
|
|
@ -47,116 +75,9 @@ pub struct LmtpConfig {
|
|||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
pub struct ImapConfig {
|
||||
pub bind_addr: SocketAddr,
|
||||
pub certs: PathBuf,
|
||||
pub key: PathBuf,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
pub struct ImapUnsecureConfig {
|
||||
pub bind_addr: SocketAddr,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
pub struct LoginStaticConfig {
|
||||
pub user_list: PathBuf,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
#[serde(tag = "storage_driver")]
|
||||
pub enum LdapStorage {
|
||||
Garage(LdapGarageConfig),
|
||||
InMemory,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
pub struct LdapGarageConfig {
|
||||
pub s3_endpoint: String,
|
||||
pub k2v_endpoint: String,
|
||||
pub aws_region: String,
|
||||
|
||||
pub aws_access_key_id_attr: String,
|
||||
pub aws_secret_access_key_attr: String,
|
||||
pub bucket_attr: Option<String>,
|
||||
pub default_bucket: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
pub struct LoginLdapConfig {
|
||||
// LDAP connection info
|
||||
pub ldap_server: String,
|
||||
#[serde(default)]
|
||||
pub pre_bind_on_login: bool,
|
||||
pub bind_dn: Option<String>,
|
||||
pub bind_password: Option<String>,
|
||||
pub search_base: String,
|
||||
|
||||
// Schema-like info required for Aerogramme's logic
|
||||
pub username_attr: String,
|
||||
#[serde(default = "default_mail_attr")]
|
||||
pub mail_attr: String,
|
||||
|
||||
// The field that will contain the crypto root thingy
|
||||
pub crypto_root_attr: String,
|
||||
|
||||
// Storage related thing
|
||||
#[serde(flatten)]
|
||||
pub storage: LdapStorage,
|
||||
}
|
||||
|
||||
// ----
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
#[serde(tag = "storage_driver")]
|
||||
pub enum StaticStorage {
|
||||
Garage(StaticGarageConfig),
|
||||
InMemory,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
pub struct StaticGarageConfig {
|
||||
pub s3_endpoint: String,
|
||||
pub k2v_endpoint: String,
|
||||
pub aws_region: String,
|
||||
|
||||
pub aws_access_key_id: String,
|
||||
pub aws_secret_access_key: String,
|
||||
pub bucket: String,
|
||||
}
|
||||
|
||||
pub type UserList = HashMap<String, UserEntry>;
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
pub struct UserEntry {
|
||||
#[serde(default)]
|
||||
pub email_addresses: Vec<String>,
|
||||
pub password: String,
|
||||
pub crypto_root: String,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub storage: StaticStorage,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
pub struct SetupEntry {
|
||||
#[serde(default)]
|
||||
pub email_addresses: Vec<String>,
|
||||
|
||||
#[serde(default)]
|
||||
pub clear_password: Option<String>,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub storage: StaticStorage,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
#[serde(tag = "role")]
|
||||
pub enum AnyConfig {
|
||||
Companion(CompanionConfig),
|
||||
Provider(ProviderConfig),
|
||||
}
|
||||
|
||||
// ---
|
||||
pub fn read_config<T: serde::de::DeserializeOwned>(config_file: PathBuf) -> Result<T> {
|
||||
pub fn read_config(config_file: PathBuf) -> Result<Config> {
|
||||
let mut file = std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.open(config_file.as_path())?;
|
||||
|
|
@ -167,18 +88,6 @@ pub fn read_config<T: serde::de::DeserializeOwned>(config_file: PathBuf) -> Resu
|
|||
Ok(toml::from_str(&config)?)
|
||||
}
|
||||
|
||||
pub fn write_config<T: Serialize>(config_file: PathBuf, config: &T) -> Result<()> {
|
||||
let mut file = std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.open(config_file.as_path())?;
|
||||
|
||||
file.write_all(toml::to_string(config)?.as_bytes())?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn default_mail_attr() -> String {
|
||||
"mail".into()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ pub fn seal(plainblob: &[u8], key: &Key) -> Result<Vec<u8>> {
|
|||
use secretbox::{gen_nonce, NONCEBYTES};
|
||||
|
||||
// Compress data using zstd
|
||||
let mut reader = plainblob;
|
||||
let mut reader = &plainblob[..];
|
||||
let zstdblob = zstd_encode(&mut reader, 0)?;
|
||||
|
||||
// Encrypt
|
||||
|
|
@ -63,5 +63,5 @@ pub fn seal_serialize<T: Serialize>(obj: T, key: &Key) -> Result<Vec<u8>> {
|
|||
.with_string_variants();
|
||||
obj.serialize(&mut se)?;
|
||||
|
||||
seal(&wr, key)
|
||||
Ok(seal(&wr, key)?)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,77 +0,0 @@
|
|||
use imap_codec::imap_types::command::FetchModifier;
|
||||
use imap_codec::imap_types::fetch::{MacroOrMessageDataItemNames, MessageDataItemName, Section};
|
||||
|
||||
/// Internal decisions based on fetched attributes
|
||||
/// passed by the client
|
||||
|
||||
pub struct AttributesProxy {
|
||||
pub attrs: Vec<MessageDataItemName<'static>>,
|
||||
}
|
||||
impl AttributesProxy {
|
||||
pub fn new(
|
||||
attrs: &MacroOrMessageDataItemNames<'static>,
|
||||
modifiers: &[FetchModifier],
|
||||
is_uid_fetch: bool,
|
||||
) -> Self {
|
||||
// Expand macros
|
||||
let mut fetch_attrs = match attrs {
|
||||
MacroOrMessageDataItemNames::Macro(m) => {
|
||||
use imap_codec::imap_types::fetch::Macro;
|
||||
use MessageDataItemName::*;
|
||||
match m {
|
||||
Macro::All => vec![Flags, InternalDate, Rfc822Size, Envelope],
|
||||
Macro::Fast => vec![Flags, InternalDate, Rfc822Size],
|
||||
Macro::Full => vec![Flags, InternalDate, Rfc822Size, Envelope, Body],
|
||||
_ => {
|
||||
tracing::error!("unimplemented macro");
|
||||
vec![]
|
||||
}
|
||||
}
|
||||
}
|
||||
MacroOrMessageDataItemNames::MessageDataItemNames(a) => a.clone(),
|
||||
};
|
||||
|
||||
// Handle uids
|
||||
if is_uid_fetch && !fetch_attrs.contains(&MessageDataItemName::Uid) {
|
||||
fetch_attrs.push(MessageDataItemName::Uid);
|
||||
}
|
||||
|
||||
// Handle inferred MODSEQ tag
|
||||
let is_changed_since = modifiers
|
||||
.iter()
|
||||
.any(|m| matches!(m, FetchModifier::ChangedSince(..)));
|
||||
if is_changed_since && !fetch_attrs.contains(&MessageDataItemName::ModSeq) {
|
||||
fetch_attrs.push(MessageDataItemName::ModSeq);
|
||||
}
|
||||
|
||||
Self { attrs: fetch_attrs }
|
||||
}
|
||||
|
||||
pub fn is_enabling_condstore(&self) -> bool {
|
||||
self.attrs
|
||||
.iter()
|
||||
.any(|x| matches!(x, MessageDataItemName::ModSeq))
|
||||
}
|
||||
|
||||
pub fn need_body(&self) -> bool {
|
||||
self.attrs.iter().any(|x| match x {
|
||||
MessageDataItemName::Body
|
||||
| MessageDataItemName::Rfc822
|
||||
| MessageDataItemName::Rfc822Text
|
||||
| MessageDataItemName::BodyStructure => true,
|
||||
|
||||
MessageDataItemName::BodyExt {
|
||||
section: Some(section),
|
||||
partial: _,
|
||||
peek: _,
|
||||
} => match section {
|
||||
Section::Header(None)
|
||||
| Section::HeaderFields(None, _)
|
||||
| Section::HeaderFieldsNot(None, _) => false,
|
||||
_ => true,
|
||||
},
|
||||
MessageDataItemName::BodyExt { .. } => true,
|
||||
_ => false,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -1,159 +0,0 @@
|
|||
use imap_codec::imap_types::command::{FetchModifier, SelectExamineModifier, StoreModifier};
|
||||
use imap_codec::imap_types::core::Vec1;
|
||||
use imap_codec::imap_types::extensions::enable::{CapabilityEnable, Utf8Kind};
|
||||
use imap_codec::imap_types::response::Capability;
|
||||
use std::collections::HashSet;
|
||||
|
||||
use crate::imap::attributes::AttributesProxy;
|
||||
|
||||
fn capability_unselect() -> Capability<'static> {
|
||||
Capability::try_from("UNSELECT").unwrap()
|
||||
}
|
||||
|
||||
fn capability_condstore() -> Capability<'static> {
|
||||
Capability::try_from("CONDSTORE").unwrap()
|
||||
}
|
||||
|
||||
fn capability_uidplus() -> Capability<'static> {
|
||||
Capability::try_from("UIDPLUS").unwrap()
|
||||
}
|
||||
|
||||
fn capability_liststatus() -> Capability<'static> {
|
||||
Capability::try_from("LIST-STATUS").unwrap()
|
||||
}
|
||||
|
||||
/*
|
||||
fn capability_qresync() -> Capability<'static> {
|
||||
Capability::try_from("QRESYNC").unwrap()
|
||||
}
|
||||
*/
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ServerCapability(HashSet<Capability<'static>>);
|
||||
|
||||
impl Default for ServerCapability {
|
||||
fn default() -> Self {
|
||||
Self(HashSet::from([
|
||||
Capability::Imap4Rev1,
|
||||
Capability::Enable,
|
||||
Capability::Move,
|
||||
Capability::LiteralPlus,
|
||||
Capability::Idle,
|
||||
capability_unselect(),
|
||||
capability_condstore(),
|
||||
capability_uidplus(),
|
||||
capability_liststatus(),
|
||||
//capability_qresync(),
|
||||
]))
|
||||
}
|
||||
}
|
||||
|
||||
impl ServerCapability {
|
||||
pub fn to_vec(&self) -> Vec1<Capability<'static>> {
|
||||
self.0
|
||||
.iter()
|
||||
.map(|v| v.clone())
|
||||
.collect::<Vec<_>>()
|
||||
.try_into()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub fn support(&self, cap: &Capability<'static>) -> bool {
|
||||
self.0.contains(cap)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub enum ClientStatus {
|
||||
NotSupportedByServer,
|
||||
Disabled,
|
||||
Enabled,
|
||||
}
|
||||
impl ClientStatus {
|
||||
pub fn is_enabled(&self) -> bool {
|
||||
matches!(self, Self::Enabled)
|
||||
}
|
||||
|
||||
pub fn enable(&self) -> Self {
|
||||
match self {
|
||||
Self::Disabled => Self::Enabled,
|
||||
other => other.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ClientCapability {
|
||||
pub condstore: ClientStatus,
|
||||
pub utf8kind: Option<Utf8Kind>,
|
||||
}
|
||||
|
||||
impl ClientCapability {
|
||||
pub fn new(sc: &ServerCapability) -> Self {
|
||||
Self {
|
||||
condstore: match sc.0.contains(&capability_condstore()) {
|
||||
true => ClientStatus::Disabled,
|
||||
_ => ClientStatus::NotSupportedByServer,
|
||||
},
|
||||
utf8kind: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn enable_condstore(&mut self) {
|
||||
self.condstore = self.condstore.enable();
|
||||
}
|
||||
|
||||
pub fn attributes_enable(&mut self, ap: &AttributesProxy) {
|
||||
if ap.is_enabling_condstore() {
|
||||
self.enable_condstore()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn fetch_modifiers_enable(&mut self, mods: &[FetchModifier]) {
|
||||
if mods
|
||||
.iter()
|
||||
.any(|x| matches!(x, FetchModifier::ChangedSince(..)))
|
||||
{
|
||||
self.enable_condstore()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn store_modifiers_enable(&mut self, mods: &[StoreModifier]) {
|
||||
if mods
|
||||
.iter()
|
||||
.any(|x| matches!(x, StoreModifier::UnchangedSince(..)))
|
||||
{
|
||||
self.enable_condstore()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn select_enable(&mut self, mods: &[SelectExamineModifier]) {
|
||||
for m in mods.iter() {
|
||||
match m {
|
||||
SelectExamineModifier::Condstore => self.enable_condstore(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn try_enable(
|
||||
&mut self,
|
||||
caps: &[CapabilityEnable<'static>],
|
||||
) -> Vec<CapabilityEnable<'static>> {
|
||||
let mut enabled = vec![];
|
||||
for cap in caps {
|
||||
match cap {
|
||||
CapabilityEnable::CondStore if matches!(self.condstore, ClientStatus::Disabled) => {
|
||||
self.condstore = ClientStatus::Enabled;
|
||||
enabled.push(cap.clone());
|
||||
}
|
||||
CapabilityEnable::Utf8(kind) if Some(kind) != self.utf8kind.as_ref() => {
|
||||
self.utf8kind = Some(kind.clone());
|
||||
enabled.push(cap.clone());
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
enabled
|
||||
}
|
||||
}
|
||||
|
|
@ -1,83 +1,76 @@
|
|||
use anyhow::Result;
|
||||
use imap_codec::imap_types::command::{Command, CommandBody};
|
||||
use imap_codec::imap_types::core::AString;
|
||||
use imap_codec::imap_types::response::Code;
|
||||
use imap_codec::imap_types::secret::Secret;
|
||||
use anyhow::{Error, Result};
|
||||
use boitalettres::proto::{res::body::Data as Body, Response};
|
||||
use imap_codec::types::command::CommandBody;
|
||||
use imap_codec::types::core::{AString, Atom};
|
||||
use imap_codec::types::response::{Capability, Code, Data, Response as ImapRes, Status};
|
||||
|
||||
use crate::imap::capability::ServerCapability;
|
||||
use crate::imap::command::anystate;
|
||||
use crate::imap::flow;
|
||||
use crate::imap::response::Response;
|
||||
use crate::login::ArcLoginProvider;
|
||||
use crate::mail::user::User;
|
||||
use crate::imap::session::InnerContext;
|
||||
|
||||
//--- dispatching
|
||||
|
||||
pub struct AnonymousContext<'a> {
|
||||
pub req: &'a Command<'static>,
|
||||
pub server_capabilities: &'a ServerCapability,
|
||||
pub login_provider: &'a ArcLoginProvider,
|
||||
}
|
||||
|
||||
pub async fn dispatch(ctx: AnonymousContext<'_>) -> Result<(Response<'static>, flow::Transition)> {
|
||||
match &ctx.req.body {
|
||||
// Any State
|
||||
CommandBody::Noop => anystate::noop_nothing(ctx.req.tag.clone()),
|
||||
CommandBody::Capability => {
|
||||
anystate::capability(ctx.req.tag.clone(), ctx.server_capabilities)
|
||||
}
|
||||
CommandBody::Logout => anystate::logout(),
|
||||
|
||||
// Specific to anonymous context (3 commands)
|
||||
CommandBody::Login { username, password } => ctx.login(username, password).await,
|
||||
CommandBody::Authenticate { .. } => {
|
||||
anystate::not_implemented(ctx.req.tag.clone(), "authenticate")
|
||||
}
|
||||
//StartTLS is not implemented for now, we will probably go full TLS.
|
||||
|
||||
// Collect other commands
|
||||
_ => anystate::wrong_state(ctx.req.tag.clone()),
|
||||
pub async fn dispatch<'a>(ctx: InnerContext<'a>) -> Result<(Response, flow::Transition)> {
|
||||
match &ctx.req.command.body {
|
||||
CommandBody::Noop => Ok((Response::ok("Noop completed.")?, flow::Transition::No)),
|
||||
CommandBody::Capability => capability(ctx).await,
|
||||
CommandBody::Logout => logout(ctx).await,
|
||||
CommandBody::Login { username, password } => login(ctx, username, password).await,
|
||||
_ => Ok((
|
||||
Response::no("This command is not available in the ANONYMOUS state.")?,
|
||||
flow::Transition::No,
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
//--- Command controllers, private
|
||||
|
||||
impl<'a> AnonymousContext<'a> {
|
||||
async fn login(
|
||||
self,
|
||||
username: &AString<'a>,
|
||||
password: &Secret<AString<'a>>,
|
||||
) -> Result<(Response<'static>, flow::Transition)> {
|
||||
let (u, p) = (
|
||||
std::str::from_utf8(username.as_ref())?,
|
||||
std::str::from_utf8(password.declassify().as_ref())?,
|
||||
);
|
||||
tracing::info!(user = %u, "command.login");
|
||||
|
||||
let creds = match self.login_provider.login(&u, &p).await {
|
||||
Err(e) => {
|
||||
tracing::debug!(error=%e, "authentication failed");
|
||||
return Ok((
|
||||
Response::build()
|
||||
.to_req(self.req)
|
||||
.message("Authentication failed")
|
||||
.no()?,
|
||||
flow::Transition::None,
|
||||
));
|
||||
}
|
||||
Ok(c) => c,
|
||||
};
|
||||
|
||||
let user = User::new(u.to_string(), creds).await?;
|
||||
|
||||
tracing::info!(username=%u, "connected");
|
||||
Ok((
|
||||
Response::build()
|
||||
.to_req(self.req)
|
||||
.code(Code::Capability(self.server_capabilities.to_vec()))
|
||||
.message("Completed")
|
||||
.ok()?,
|
||||
flow::Transition::Authenticate(user),
|
||||
))
|
||||
}
|
||||
async fn capability<'a>(ctx: InnerContext<'a>) -> Result<(Response, flow::Transition)> {
|
||||
let capabilities = vec![Capability::Imap4Rev1, Capability::Idle];
|
||||
let res = Response::ok("Server capabilities")?.with_body(Data::Capability(capabilities));
|
||||
Ok((res, flow::Transition::No))
|
||||
}
|
||||
|
||||
async fn login<'a>(
|
||||
ctx: InnerContext<'a>,
|
||||
username: &AString,
|
||||
password: &AString,
|
||||
) -> Result<(Response, flow::Transition)> {
|
||||
let (u, p) = (
|
||||
String::try_from(username.clone())?,
|
||||
String::try_from(password.clone())?,
|
||||
);
|
||||
tracing::info!(user = %u, "command.login");
|
||||
|
||||
let creds = match ctx.login.login(&u, &p).await {
|
||||
Err(e) => {
|
||||
tracing::debug!(error=%e, "authentication failed");
|
||||
return Ok((Response::no("Authentication failed")?, flow::Transition::No));
|
||||
}
|
||||
Ok(c) => c,
|
||||
};
|
||||
|
||||
let user = flow::User {
|
||||
creds,
|
||||
name: u.clone(),
|
||||
};
|
||||
|
||||
tracing::info!(username=%u, "connected");
|
||||
Ok((
|
||||
Response::ok("Completed")?,
|
||||
flow::Transition::Authenticate(user),
|
||||
))
|
||||
}
|
||||
// C: 10 logout
|
||||
// S: * BYE Logging out
|
||||
// S: 10 OK Logout completed.
|
||||
async fn logout<'a>(ctx: InnerContext<'a>) -> Result<(Response, flow::Transition)> {
|
||||
// @FIXME we should implement From<Vec<Status>> and From<Vec<ImapStatus>> in
|
||||
// boitalettres/src/proto/res/body.rs
|
||||
Ok((
|
||||
Response::ok("Logout completed")?.with_body(vec![Body::Status(
|
||||
Status::bye(None, "Logging out")
|
||||
.map_err(|e| Error::msg(e).context("Unable to generate IMAP status"))?,
|
||||
)]),
|
||||
flow::Transition::Logout,
|
||||
))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,54 +0,0 @@
|
|||
use anyhow::Result;
|
||||
use imap_codec::imap_types::core::Tag;
|
||||
use imap_codec::imap_types::response::Data;
|
||||
|
||||
use crate::imap::capability::ServerCapability;
|
||||
use crate::imap::flow;
|
||||
use crate::imap::response::Response;
|
||||
|
||||
pub(crate) fn capability(
|
||||
tag: Tag<'static>,
|
||||
cap: &ServerCapability,
|
||||
) -> Result<(Response<'static>, flow::Transition)> {
|
||||
let res = Response::build()
|
||||
.tag(tag)
|
||||
.message("Server capabilities")
|
||||
.data(Data::Capability(cap.to_vec()))
|
||||
.ok()?;
|
||||
|
||||
Ok((res, flow::Transition::None))
|
||||
}
|
||||
|
||||
pub(crate) fn noop_nothing(tag: Tag<'static>) -> Result<(Response<'static>, flow::Transition)> {
|
||||
Ok((
|
||||
Response::build().tag(tag).message("Noop completed.").ok()?,
|
||||
flow::Transition::None,
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) fn logout() -> Result<(Response<'static>, flow::Transition)> {
|
||||
Ok((Response::bye()?, flow::Transition::Logout))
|
||||
}
|
||||
|
||||
pub(crate) fn not_implemented<'a>(
|
||||
tag: Tag<'a>,
|
||||
what: &str,
|
||||
) -> Result<(Response<'a>, flow::Transition)> {
|
||||
Ok((
|
||||
Response::build()
|
||||
.tag(tag)
|
||||
.message(format!("Command not implemented {}", what))
|
||||
.bad()?,
|
||||
flow::Transition::None,
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) fn wrong_state(tag: Tag<'static>) -> Result<(Response<'static>, flow::Transition)> {
|
||||
Ok((
|
||||
Response::build()
|
||||
.tag(tag)
|
||||
.message("Command not authorized in this state")
|
||||
.bad()?,
|
||||
flow::Transition::None,
|
||||
))
|
||||
}
|
||||
|
|
@ -1,442 +1,66 @@
|
|||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
use thiserror::Error;
|
||||
use anyhow::{anyhow, Error, Result};
|
||||
use boitalettres::proto::{res::body::Data as Body, Response};
|
||||
use imap_codec::types::command::CommandBody;
|
||||
use imap_codec::types::core::Atom;
|
||||
use imap_codec::types::flag::Flag;
|
||||
use imap_codec::types::mailbox::{ListMailbox, Mailbox as MailboxCodec};
|
||||
use imap_codec::types::response::{Code, Data, Response as ImapRes, Status};
|
||||
|
||||
use anyhow::{anyhow, bail, Result};
|
||||
use imap_codec::imap_types::command::{
|
||||
Command, CommandBody, ListReturnItem, SelectExamineModifier,
|
||||
};
|
||||
use imap_codec::imap_types::core::{Atom, Literal, QuotedChar, Vec1};
|
||||
use imap_codec::imap_types::datetime::DateTime;
|
||||
use imap_codec::imap_types::extensions::enable::CapabilityEnable;
|
||||
use imap_codec::imap_types::flag::{Flag, FlagNameAttribute};
|
||||
use imap_codec::imap_types::mailbox::{ListMailbox, Mailbox as MailboxCodec};
|
||||
use imap_codec::imap_types::response::{Code, CodeOther, Data};
|
||||
use imap_codec::imap_types::status::{StatusDataItem, StatusDataItemName};
|
||||
|
||||
use crate::imap::capability::{ClientCapability, ServerCapability};
|
||||
use crate::imap::command::{anystate, MailboxName};
|
||||
use crate::imap::command::anonymous;
|
||||
use crate::imap::flow;
|
||||
use crate::imap::mailbox_view::{MailboxView, UpdateParameters};
|
||||
use crate::imap::response::Response;
|
||||
use crate::imap::Body;
|
||||
use crate::imap::session::InnerContext;
|
||||
use crate::mail::Mailbox;
|
||||
|
||||
use crate::mail::uidindex::*;
|
||||
use crate::mail::user::{User, MAILBOX_HIERARCHY_DELIMITER as MBX_HIER_DELIM_RAW};
|
||||
use crate::mail::IMF;
|
||||
|
||||
pub struct AuthenticatedContext<'a> {
|
||||
pub req: &'a Command<'static>,
|
||||
pub server_capabilities: &'a ServerCapability,
|
||||
pub client_capabilities: &'a mut ClientCapability,
|
||||
pub user: &'a Arc<User>,
|
||||
}
|
||||
const DEFAULT_FLAGS: [Flag; 5] = [
|
||||
Flag::Seen,
|
||||
Flag::Answered,
|
||||
Flag::Flagged,
|
||||
Flag::Deleted,
|
||||
Flag::Draft,
|
||||
];
|
||||
|
||||
pub async fn dispatch<'a>(
|
||||
mut ctx: AuthenticatedContext<'a>,
|
||||
) -> Result<(Response<'static>, flow::Transition)> {
|
||||
match &ctx.req.body {
|
||||
// Any state
|
||||
CommandBody::Noop => anystate::noop_nothing(ctx.req.tag.clone()),
|
||||
CommandBody::Capability => {
|
||||
anystate::capability(ctx.req.tag.clone(), ctx.server_capabilities)
|
||||
}
|
||||
CommandBody::Logout => anystate::logout(),
|
||||
inner: InnerContext<'a>,
|
||||
user: &'a flow::User,
|
||||
) -> Result<(Response, flow::Transition)> {
|
||||
let ctx = StateContext { user, inner };
|
||||
|
||||
// Specific to this state (11 commands)
|
||||
CommandBody::Create { mailbox } => ctx.create(mailbox).await,
|
||||
CommandBody::Delete { mailbox } => ctx.delete(mailbox).await,
|
||||
CommandBody::Rename { from, to } => ctx.rename(from, to).await,
|
||||
match &ctx.inner.req.command.body {
|
||||
CommandBody::Lsub {
|
||||
reference,
|
||||
mailbox_wildcard,
|
||||
} => ctx.list(reference, mailbox_wildcard, &[], true).await,
|
||||
} => ctx.lsub(reference, mailbox_wildcard).await,
|
||||
CommandBody::List {
|
||||
reference,
|
||||
mailbox_wildcard,
|
||||
r#return,
|
||||
} => ctx.list(reference, mailbox_wildcard, r#return, false).await,
|
||||
CommandBody::Status {
|
||||
mailbox,
|
||||
item_names,
|
||||
} => ctx.status(mailbox, item_names).await,
|
||||
CommandBody::Subscribe { mailbox } => ctx.subscribe(mailbox).await,
|
||||
CommandBody::Unsubscribe { mailbox } => ctx.unsubscribe(mailbox).await,
|
||||
CommandBody::Select { mailbox, modifiers } => ctx.select(mailbox, modifiers).await,
|
||||
CommandBody::Examine { mailbox, modifiers } => ctx.examine(mailbox, modifiers).await,
|
||||
CommandBody::Append {
|
||||
mailbox,
|
||||
flags,
|
||||
date,
|
||||
message,
|
||||
} => ctx.append(mailbox, flags, date, message).await,
|
||||
|
||||
// rfc5161 ENABLE
|
||||
CommandBody::Enable { capabilities } => ctx.enable(capabilities),
|
||||
|
||||
// Collect other commands
|
||||
_ => anystate::wrong_state(ctx.req.tag.clone()),
|
||||
} => ctx.list(reference, mailbox_wildcard).await,
|
||||
CommandBody::Select { mailbox } => ctx.select(mailbox).await,
|
||||
_ => anonymous::dispatch(ctx.inner).await,
|
||||
}
|
||||
}
|
||||
|
||||
// --- PRIVATE ---
|
||||
impl<'a> AuthenticatedContext<'a> {
|
||||
async fn create(
|
||||
self,
|
||||
mailbox: &MailboxCodec<'a>,
|
||||
) -> Result<(Response<'static>, flow::Transition)> {
|
||||
let name = match mailbox {
|
||||
MailboxCodec::Inbox => {
|
||||
return Ok((
|
||||
Response::build()
|
||||
.to_req(self.req)
|
||||
.message("Cannot create INBOX")
|
||||
.bad()?,
|
||||
flow::Transition::None,
|
||||
));
|
||||
}
|
||||
MailboxCodec::Other(aname) => std::str::from_utf8(aname.as_ref())?,
|
||||
};
|
||||
|
||||
match self.user.create_mailbox(&name).await {
|
||||
Ok(()) => Ok((
|
||||
Response::build()
|
||||
.to_req(self.req)
|
||||
.message("CREATE complete")
|
||||
.ok()?,
|
||||
flow::Transition::None,
|
||||
)),
|
||||
Err(e) => Ok((
|
||||
Response::build()
|
||||
.to_req(self.req)
|
||||
.message(&e.to_string())
|
||||
.no()?,
|
||||
flow::Transition::None,
|
||||
)),
|
||||
}
|
||||
}
|
||||
struct StateContext<'a> {
|
||||
inner: InnerContext<'a>,
|
||||
user: &'a flow::User,
|
||||
}
|
||||
|
||||
async fn delete(
|
||||
self,
|
||||
mailbox: &MailboxCodec<'a>,
|
||||
) -> Result<(Response<'static>, flow::Transition)> {
|
||||
let name: &str = MailboxName(mailbox).try_into()?;
|
||||
|
||||
match self.user.delete_mailbox(&name).await {
|
||||
Ok(()) => Ok((
|
||||
Response::build()
|
||||
.to_req(self.req)
|
||||
.message("DELETE complete")
|
||||
.ok()?,
|
||||
flow::Transition::None,
|
||||
)),
|
||||
Err(e) => Ok((
|
||||
Response::build()
|
||||
.to_req(self.req)
|
||||
.message(e.to_string())
|
||||
.no()?,
|
||||
flow::Transition::None,
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn rename(
|
||||
self,
|
||||
from: &MailboxCodec<'a>,
|
||||
to: &MailboxCodec<'a>,
|
||||
) -> Result<(Response<'static>, flow::Transition)> {
|
||||
let name: &str = MailboxName(from).try_into()?;
|
||||
let new_name: &str = MailboxName(to).try_into()?;
|
||||
|
||||
match self.user.rename_mailbox(&name, &new_name).await {
|
||||
Ok(()) => Ok((
|
||||
Response::build()
|
||||
.to_req(self.req)
|
||||
.message("RENAME complete")
|
||||
.ok()?,
|
||||
flow::Transition::None,
|
||||
)),
|
||||
Err(e) => Ok((
|
||||
Response::build()
|
||||
.to_req(self.req)
|
||||
.message(e.to_string())
|
||||
.no()?,
|
||||
flow::Transition::None,
|
||||
)),
|
||||
}
|
||||
impl<'a> StateContext<'a> {
|
||||
async fn lsub(
|
||||
&self,
|
||||
reference: &MailboxCodec,
|
||||
mailbox_wildcard: &ListMailbox,
|
||||
) -> Result<(Response, flow::Transition)> {
|
||||
Ok((Response::bad("Not implemented")?, flow::Transition::No))
|
||||
}
|
||||
|
||||
async fn list(
|
||||
&mut self,
|
||||
reference: &MailboxCodec<'a>,
|
||||
mailbox_wildcard: &ListMailbox<'a>,
|
||||
must_return: &[ListReturnItem],
|
||||
is_lsub: bool,
|
||||
) -> Result<(Response<'static>, flow::Transition)> {
|
||||
let mbx_hier_delim: QuotedChar = QuotedChar::unvalidated(MBX_HIER_DELIM_RAW);
|
||||
|
||||
let reference: &str = MailboxName(reference).try_into()?;
|
||||
if !reference.is_empty() {
|
||||
return Ok((
|
||||
Response::build()
|
||||
.to_req(self.req)
|
||||
.message("References not supported")
|
||||
.bad()?,
|
||||
flow::Transition::None,
|
||||
));
|
||||
}
|
||||
|
||||
let status_item_names = must_return.iter().find_map(|m| match m {
|
||||
ListReturnItem::Status(v) => Some(v),
|
||||
_ => None,
|
||||
});
|
||||
|
||||
// @FIXME would probably need a rewrite to better use the imap_codec library
|
||||
let wildcard = match mailbox_wildcard {
|
||||
ListMailbox::Token(v) => std::str::from_utf8(v.as_ref())?,
|
||||
ListMailbox::String(v) => std::str::from_utf8(v.as_ref())?,
|
||||
};
|
||||
if wildcard.is_empty() {
|
||||
if is_lsub {
|
||||
return Ok((
|
||||
Response::build()
|
||||
.to_req(self.req)
|
||||
.message("LSUB complete")
|
||||
.data(Data::Lsub {
|
||||
items: vec![],
|
||||
delimiter: Some(mbx_hier_delim),
|
||||
mailbox: "".try_into().unwrap(),
|
||||
})
|
||||
.ok()?,
|
||||
flow::Transition::None,
|
||||
));
|
||||
} else {
|
||||
return Ok((
|
||||
Response::build()
|
||||
.to_req(self.req)
|
||||
.message("LIST complete")
|
||||
.data(Data::List {
|
||||
items: vec![],
|
||||
delimiter: Some(mbx_hier_delim),
|
||||
mailbox: "".try_into().unwrap(),
|
||||
})
|
||||
.ok()?,
|
||||
flow::Transition::None,
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
let mailboxes = self.user.list_mailboxes().await?;
|
||||
let mut vmailboxes = BTreeMap::new();
|
||||
for mb in mailboxes.iter() {
|
||||
for (i, _) in mb.match_indices(MBX_HIER_DELIM_RAW) {
|
||||
if i > 0 {
|
||||
let smb = &mb[..i];
|
||||
vmailboxes.entry(smb).or_insert(false);
|
||||
}
|
||||
}
|
||||
vmailboxes.insert(mb, true);
|
||||
}
|
||||
|
||||
let mut ret = vec![];
|
||||
for (mb, is_real) in vmailboxes.iter() {
|
||||
if matches_wildcard(&wildcard, mb) {
|
||||
let mailbox: MailboxCodec = mb
|
||||
.to_string()
|
||||
.try_into()
|
||||
.map_err(|_| anyhow!("invalid mailbox name"))?;
|
||||
let mut items = vec![FlagNameAttribute::from(Atom::unvalidated("Subscribed"))];
|
||||
|
||||
// Decoration
|
||||
if !*is_real {
|
||||
items.push(FlagNameAttribute::Noselect);
|
||||
} else {
|
||||
match *mb {
|
||||
"Drafts" => items.push(Atom::unvalidated("Drafts").into()),
|
||||
"Archive" => items.push(Atom::unvalidated("Archive").into()),
|
||||
"Sent" => items.push(Atom::unvalidated("Sent").into()),
|
||||
"Trash" => items.push(Atom::unvalidated("Trash").into()),
|
||||
_ => (),
|
||||
};
|
||||
}
|
||||
|
||||
// Result type
|
||||
if is_lsub {
|
||||
ret.push(Data::Lsub {
|
||||
items,
|
||||
delimiter: Some(mbx_hier_delim),
|
||||
mailbox: mailbox.clone(),
|
||||
});
|
||||
} else {
|
||||
ret.push(Data::List {
|
||||
items,
|
||||
delimiter: Some(mbx_hier_delim),
|
||||
mailbox: mailbox.clone(),
|
||||
});
|
||||
}
|
||||
|
||||
// Also collect status
|
||||
if let Some(sin) = status_item_names {
|
||||
let ret_attrs = match self.status_items(mb, sin).await {
|
||||
Ok(a) => a,
|
||||
Err(e) => {
|
||||
tracing::error!(err=?e, mailbox=%mb, "Unable to fetch status for mailbox");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let data = Data::Status {
|
||||
mailbox,
|
||||
items: ret_attrs.into(),
|
||||
};
|
||||
|
||||
ret.push(data);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let msg = if is_lsub {
|
||||
"LSUB completed"
|
||||
} else {
|
||||
"LIST completed"
|
||||
};
|
||||
Ok((
|
||||
Response::build()
|
||||
.to_req(self.req)
|
||||
.message(msg)
|
||||
.many_data(ret)
|
||||
.ok()?,
|
||||
flow::Transition::None,
|
||||
))
|
||||
}
|
||||
|
||||
async fn status(
|
||||
&mut self,
|
||||
mailbox: &MailboxCodec<'static>,
|
||||
attributes: &[StatusDataItemName],
|
||||
) -> Result<(Response<'static>, flow::Transition)> {
|
||||
let name: &str = MailboxName(mailbox).try_into()?;
|
||||
|
||||
let ret_attrs = match self.status_items(name, attributes).await {
|
||||
Ok(v) => v,
|
||||
Err(e) => match e.downcast_ref::<CommandError>() {
|
||||
Some(CommandError::MailboxNotFound) => {
|
||||
return Ok((
|
||||
Response::build()
|
||||
.to_req(self.req)
|
||||
.message("Mailbox does not exist")
|
||||
.no()?,
|
||||
flow::Transition::None,
|
||||
))
|
||||
}
|
||||
_ => return Err(e.into()),
|
||||
},
|
||||
};
|
||||
|
||||
let data = Data::Status {
|
||||
mailbox: mailbox.clone(),
|
||||
items: ret_attrs.into(),
|
||||
};
|
||||
|
||||
Ok((
|
||||
Response::build()
|
||||
.to_req(self.req)
|
||||
.message("STATUS completed")
|
||||
.data(data)
|
||||
.ok()?,
|
||||
flow::Transition::None,
|
||||
))
|
||||
}
|
||||
|
||||
async fn status_items(
|
||||
&mut self,
|
||||
name: &str,
|
||||
attributes: &[StatusDataItemName],
|
||||
) -> Result<Vec<StatusDataItem>> {
|
||||
let mb_opt = self.user.open_mailbox(name).await?;
|
||||
let mb = match mb_opt {
|
||||
Some(mb) => mb,
|
||||
None => return Err(CommandError::MailboxNotFound.into()),
|
||||
};
|
||||
|
||||
let view = MailboxView::new(mb, self.client_capabilities.condstore.is_enabled()).await;
|
||||
|
||||
let mut ret_attrs = vec![];
|
||||
for attr in attributes.iter() {
|
||||
ret_attrs.push(match attr {
|
||||
StatusDataItemName::Messages => StatusDataItem::Messages(view.exists()?),
|
||||
StatusDataItemName::Unseen => StatusDataItem::Unseen(view.unseen_count() as u32),
|
||||
StatusDataItemName::Recent => StatusDataItem::Recent(view.recent()?),
|
||||
StatusDataItemName::UidNext => StatusDataItem::UidNext(view.uidnext()),
|
||||
StatusDataItemName::UidValidity => {
|
||||
StatusDataItem::UidValidity(view.uidvalidity())
|
||||
}
|
||||
StatusDataItemName::Deleted => {
|
||||
bail!("quota not implemented, can't return deleted elements waiting for EXPUNGE");
|
||||
},
|
||||
StatusDataItemName::DeletedStorage => {
|
||||
bail!("quota not implemented, can't return freed storage after EXPUNGE will be run");
|
||||
},
|
||||
StatusDataItemName::HighestModSeq => {
|
||||
self.client_capabilities.enable_condstore();
|
||||
StatusDataItem::HighestModSeq(view.highestmodseq().get())
|
||||
},
|
||||
});
|
||||
}
|
||||
Ok(ret_attrs)
|
||||
}
|
||||
|
||||
async fn subscribe(
|
||||
self,
|
||||
mailbox: &MailboxCodec<'a>,
|
||||
) -> Result<(Response<'static>, flow::Transition)> {
|
||||
let name: &str = MailboxName(mailbox).try_into()?;
|
||||
|
||||
if self.user.has_mailbox(&name).await? {
|
||||
Ok((
|
||||
Response::build()
|
||||
.to_req(self.req)
|
||||
.message("SUBSCRIBE complete")
|
||||
.ok()?,
|
||||
flow::Transition::None,
|
||||
))
|
||||
} else {
|
||||
Ok((
|
||||
Response::build()
|
||||
.to_req(self.req)
|
||||
.message(format!("Mailbox {} does not exist", name))
|
||||
.bad()?,
|
||||
flow::Transition::None,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
async fn unsubscribe(
|
||||
self,
|
||||
mailbox: &MailboxCodec<'a>,
|
||||
) -> Result<(Response<'static>, flow::Transition)> {
|
||||
let name: &str = MailboxName(mailbox).try_into()?;
|
||||
|
||||
if self.user.has_mailbox(&name).await? {
|
||||
Ok((
|
||||
Response::build()
|
||||
.to_req(self.req)
|
||||
.message(format!(
|
||||
"Cannot unsubscribe from mailbox {}: not supported by Aerogramme",
|
||||
name
|
||||
))
|
||||
.bad()?,
|
||||
flow::Transition::None,
|
||||
))
|
||||
} else {
|
||||
Ok((
|
||||
Response::build()
|
||||
.to_req(self.req)
|
||||
.message(format!("Mailbox {} does not exist", name))
|
||||
.no()?,
|
||||
flow::Transition::None,
|
||||
))
|
||||
}
|
||||
&self,
|
||||
reference: &MailboxCodec,
|
||||
mailbox_wildcard: &ListMailbox,
|
||||
) -> Result<(Response, flow::Transition)> {
|
||||
Ok((Response::bad("Not implemented")?, flow::Transition::No))
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -454,12 +78,6 @@ impl<'a> AuthenticatedContext<'a> {
|
|||
S: A142 OK [READ-WRITE] SELECT completed
|
||||
|
||||
--- a mailbox with no unseen message -> no unseen entry
|
||||
NOTES:
|
||||
RFC3501 (imap4rev1) says if there is no OK [UNSEEN] response, client must make no assumption,
|
||||
it is therefore correct to not return it even if there are unseen messages
|
||||
RFC9051 (imap4rev2) says that OK [UNSEEN] responses are deprecated after SELECT and EXAMINE
|
||||
For Aerogramme, we just don't send the OK [UNSEEN], it's correct to do in both specifications.
|
||||
|
||||
|
||||
20 select "INBOX.achats"
|
||||
* FLAGS (\Answered \Flagged \Deleted \Seen \Draft $Forwarded JUNK $label1)
|
||||
|
|
@ -473,210 +91,63 @@ impl<'a> AuthenticatedContext<'a> {
|
|||
|
||||
* TRACE END ---
|
||||
*/
|
||||
async fn select(
|
||||
self,
|
||||
mailbox: &MailboxCodec<'a>,
|
||||
modifiers: &[SelectExamineModifier],
|
||||
) -> Result<(Response<'static>, flow::Transition)> {
|
||||
self.client_capabilities.select_enable(modifiers);
|
||||
async fn select(&self, mailbox: &MailboxCodec) -> Result<(Response, flow::Transition)> {
|
||||
let name = String::try_from(mailbox.clone())?;
|
||||
|
||||
let name: &str = MailboxName(mailbox).try_into()?;
|
||||
let mut mb = Mailbox::new(&self.user.creds, name.clone())?;
|
||||
tracing::info!(username=%self.user.name, mailbox=%name, "mailbox.selected");
|
||||
|
||||
let mb_opt = self.user.open_mailbox(&name).await?;
|
||||
let mb = match mb_opt {
|
||||
Some(mb) => mb,
|
||||
None => {
|
||||
return Ok((
|
||||
Response::build()
|
||||
.to_req(self.req)
|
||||
.message("Mailbox does not exist")
|
||||
.no()?,
|
||||
flow::Transition::None,
|
||||
))
|
||||
}
|
||||
};
|
||||
tracing::info!(username=%self.user.username, mailbox=%name, "mailbox.selected");
|
||||
let sum = mb.summary().await?;
|
||||
tracing::trace!(summary=%sum, "mailbox.summary");
|
||||
|
||||
let mb = MailboxView::new(mb, self.client_capabilities.condstore.is_enabled()).await;
|
||||
let data = mb.summary()?;
|
||||
let mut res = Vec::<Body>::new();
|
||||
|
||||
res.push(Body::Data(Data::Exists(sum.exists)));
|
||||
|
||||
res.push(Body::Data(Data::Recent(sum.recent)));
|
||||
|
||||
let mut flags: Vec<Flag> = sum.flags.map(|f| match f.chars().next() {
|
||||
Some('\\') => None,
|
||||
Some('$') if f == "$unseen" => None,
|
||||
Some(_) => match Atom::try_from(f.clone()) {
|
||||
Err(_) => {
|
||||
tracing::error!(username=%self.user.name, mailbox=%name, flag=%f, "Unable to encode flag as IMAP atom");
|
||||
None
|
||||
},
|
||||
Ok(a) => Some(Flag::Keyword(a)),
|
||||
},
|
||||
None => None,
|
||||
}).flatten().collect();
|
||||
flags.extend_from_slice(&DEFAULT_FLAGS);
|
||||
|
||||
res.push(Body::Data(Data::Flags(flags.clone())));
|
||||
|
||||
let uid_validity = Status::ok(None, Some(Code::UidValidity(sum.validity)), "UIDs valid")
|
||||
.map_err(Error::msg)?;
|
||||
res.push(Body::Status(uid_validity));
|
||||
|
||||
let next_uid = Status::ok(None, Some(Code::UidNext(sum.next)), "Predict next UID")
|
||||
.map_err(Error::msg)?;
|
||||
res.push(Body::Status(next_uid));
|
||||
|
||||
if let Some(unseen) = sum.unseen {
|
||||
let status_unseen =
|
||||
Status::ok(None, Some(Code::Unseen(unseen.clone())), "First unseen UID")
|
||||
.map_err(Error::msg)?;
|
||||
res.push(Body::Status(status_unseen));
|
||||
}
|
||||
|
||||
flags.push(Flag::Permanent);
|
||||
let permanent_flags =
|
||||
Status::ok(None, Some(Code::PermanentFlags(flags)), "Flags permitted")
|
||||
.map_err(Error::msg)?;
|
||||
res.push(Body::Status(permanent_flags));
|
||||
|
||||
Ok((
|
||||
Response::build()
|
||||
.message("Select completed")
|
||||
.to_req(self.req)
|
||||
.code(Code::ReadWrite)
|
||||
.set_body(data)
|
||||
.ok()?,
|
||||
flow::Transition::Select(mb, flow::MailboxPerm::ReadWrite),
|
||||
Response::ok("Select completed")?
|
||||
.with_extra_code(Code::ReadWrite)
|
||||
.with_body(res),
|
||||
flow::Transition::Select(mb),
|
||||
))
|
||||
}
|
||||
|
||||
async fn examine(
|
||||
self,
|
||||
mailbox: &MailboxCodec<'a>,
|
||||
modifiers: &[SelectExamineModifier],
|
||||
) -> Result<(Response<'static>, flow::Transition)> {
|
||||
self.client_capabilities.select_enable(modifiers);
|
||||
|
||||
let name: &str = MailboxName(mailbox).try_into()?;
|
||||
|
||||
let mb_opt = self.user.open_mailbox(&name).await?;
|
||||
let mb = match mb_opt {
|
||||
Some(mb) => mb,
|
||||
None => {
|
||||
return Ok((
|
||||
Response::build()
|
||||
.to_req(self.req)
|
||||
.message("Mailbox does not exist")
|
||||
.no()?,
|
||||
flow::Transition::None,
|
||||
))
|
||||
}
|
||||
};
|
||||
tracing::info!(username=%self.user.username, mailbox=%name, "mailbox.examined");
|
||||
|
||||
let mb = MailboxView::new(mb, self.client_capabilities.condstore.is_enabled()).await;
|
||||
let data = mb.summary()?;
|
||||
|
||||
Ok((
|
||||
Response::build()
|
||||
.to_req(self.req)
|
||||
.message("Examine completed")
|
||||
.code(Code::ReadOnly)
|
||||
.set_body(data)
|
||||
.ok()?,
|
||||
flow::Transition::Select(mb, flow::MailboxPerm::ReadOnly),
|
||||
))
|
||||
}
|
||||
|
||||
//@FIXME we should write a specific version for the "selected" state
|
||||
//that returns some unsollicited responses
|
||||
async fn append(
|
||||
self,
|
||||
mailbox: &MailboxCodec<'a>,
|
||||
flags: &[Flag<'a>],
|
||||
date: &Option<DateTime>,
|
||||
message: &Literal<'a>,
|
||||
) -> Result<(Response<'static>, flow::Transition)> {
|
||||
let append_tag = self.req.tag.clone();
|
||||
match self.append_internal(mailbox, flags, date, message).await {
|
||||
Ok((_mb_view, uidvalidity, uid, _modseq)) => Ok((
|
||||
Response::build()
|
||||
.tag(append_tag)
|
||||
.message("APPEND completed")
|
||||
.code(Code::Other(CodeOther::unvalidated(
|
||||
format!("APPENDUID {} {}", uidvalidity, uid).into_bytes(),
|
||||
)))
|
||||
.ok()?,
|
||||
flow::Transition::None,
|
||||
)),
|
||||
Err(e) => Ok((
|
||||
Response::build()
|
||||
.tag(append_tag)
|
||||
.message(e.to_string())
|
||||
.no()?,
|
||||
flow::Transition::None,
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
fn enable(
|
||||
self,
|
||||
cap_enable: &Vec1<CapabilityEnable<'static>>,
|
||||
) -> Result<(Response<'static>, flow::Transition)> {
|
||||
let mut response_builder = Response::build().to_req(self.req);
|
||||
let capabilities = self.client_capabilities.try_enable(cap_enable.as_ref());
|
||||
if capabilities.len() > 0 {
|
||||
response_builder = response_builder.data(Data::Enabled { capabilities });
|
||||
}
|
||||
Ok((
|
||||
response_builder.message("ENABLE completed").ok()?,
|
||||
flow::Transition::None,
|
||||
))
|
||||
}
|
||||
|
||||
//@FIXME should be refactored and integrated to the mailbox view
|
||||
pub(crate) async fn append_internal(
|
||||
self,
|
||||
mailbox: &MailboxCodec<'a>,
|
||||
flags: &[Flag<'a>],
|
||||
date: &Option<DateTime>,
|
||||
message: &Literal<'a>,
|
||||
) -> Result<(MailboxView, ImapUidvalidity, ImapUid, ModSeq)> {
|
||||
let name: &str = MailboxName(mailbox).try_into()?;
|
||||
|
||||
let mb_opt = self.user.open_mailbox(&name).await?;
|
||||
let mb = match mb_opt {
|
||||
Some(mb) => mb,
|
||||
None => bail!("Mailbox does not exist"),
|
||||
};
|
||||
let mut view = MailboxView::new(mb, self.client_capabilities.condstore.is_enabled()).await;
|
||||
|
||||
if date.is_some() {
|
||||
tracing::warn!("Cannot set date when appending message");
|
||||
}
|
||||
|
||||
let msg =
|
||||
IMF::try_from(message.data()).map_err(|_| anyhow!("Could not parse e-mail message"))?;
|
||||
let flags = flags.iter().map(|x| x.to_string()).collect::<Vec<_>>();
|
||||
// TODO: filter allowed flags? ping @Quentin
|
||||
|
||||
let (uidvalidity, uid, modseq) =
|
||||
view.internal.mailbox.append(msg, None, &flags[..]).await?;
|
||||
//let unsollicited = view.update(UpdateParameters::default()).await?;
|
||||
|
||||
Ok((view, uidvalidity, uid, modseq))
|
||||
}
|
||||
}
|
||||
|
||||
fn matches_wildcard(wildcard: &str, name: &str) -> bool {
|
||||
let wildcard = wildcard.chars().collect::<Vec<char>>();
|
||||
let name = name.chars().collect::<Vec<char>>();
|
||||
|
||||
let mut matches = vec![vec![false; wildcard.len() + 1]; name.len() + 1];
|
||||
|
||||
for i in 0..=name.len() {
|
||||
for j in 0..=wildcard.len() {
|
||||
matches[i][j] = (i == 0 && j == 0)
|
||||
|| (j > 0
|
||||
&& matches[i][j - 1]
|
||||
&& (wildcard[j - 1] == '%' || wildcard[j - 1] == '*'))
|
||||
|| (i > 0
|
||||
&& j > 0
|
||||
&& matches[i - 1][j - 1]
|
||||
&& wildcard[j - 1] == name[i - 1]
|
||||
&& wildcard[j - 1] != '%'
|
||||
&& wildcard[j - 1] != '*')
|
||||
|| (i > 0
|
||||
&& j > 0
|
||||
&& matches[i - 1][j]
|
||||
&& (wildcard[j - 1] == '*'
|
||||
|| (wildcard[j - 1] == '%' && name[i - 1] != MBX_HIER_DELIM_RAW)));
|
||||
}
|
||||
}
|
||||
|
||||
matches[name.len()][wildcard.len()]
|
||||
}
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum CommandError {
|
||||
#[error("Mailbox not found")]
|
||||
MailboxNotFound,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_wildcard_matches() {
|
||||
assert!(matches_wildcard("INBOX", "INBOX"));
|
||||
assert!(matches_wildcard("*", "INBOX"));
|
||||
assert!(matches_wildcard("%", "INBOX"));
|
||||
assert!(!matches_wildcard("%", "Test.Azerty"));
|
||||
assert!(!matches_wildcard("INBOX.*", "INBOX"));
|
||||
assert!(matches_wildcard("Sent.*", "Sent.A"));
|
||||
assert!(matches_wildcard("Sent.*", "Sent.A.B"));
|
||||
assert!(!matches_wildcard("Sent.%", "Sent.A.B"));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,20 +1,3 @@
|
|||
pub mod anonymous;
|
||||
pub mod anystate;
|
||||
pub mod authenticated;
|
||||
pub mod selected;
|
||||
|
||||
use crate::mail::user::INBOX;
|
||||
use imap_codec::imap_types::mailbox::Mailbox as MailboxCodec;
|
||||
|
||||
/// Convert an IMAP mailbox name/identifier representation
|
||||
/// to an utf-8 string that is used internally in Aerogramme
|
||||
struct MailboxName<'a>(&'a MailboxCodec<'a>);
|
||||
impl<'a> TryInto<&'a str> for MailboxName<'a> {
|
||||
type Error = std::str::Utf8Error;
|
||||
fn try_into(self) -> Result<&'a str, Self::Error> {
|
||||
match self.0 {
|
||||
MailboxCodec::Inbox => Ok(INBOX),
|
||||
MailboxCodec::Other(aname) => Ok(std::str::from_utf8(aname.as_ref())?),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,424 +1,52 @@
|
|||
use std::num::NonZeroU64;
|
||||
use std::sync::Arc;
|
||||
use anyhow::{Error, Result};
|
||||
use boitalettres::proto::Response;
|
||||
use imap_codec::types::command::CommandBody;
|
||||
use imap_codec::types::core::Tag;
|
||||
use imap_codec::types::fetch_attributes::MacroOrFetchAttributes;
|
||||
use imap_codec::types::response::{Response as ImapRes, Status};
|
||||
use imap_codec::types::sequence::SequenceSet;
|
||||
|
||||
use anyhow::Result;
|
||||
use imap_codec::imap_types::command::{Command, CommandBody, FetchModifier, StoreModifier};
|
||||
use imap_codec::imap_types::core::{Charset, Vec1};
|
||||
use imap_codec::imap_types::fetch::MacroOrMessageDataItemNames;
|
||||
use imap_codec::imap_types::flag::{Flag, StoreResponse, StoreType};
|
||||
use imap_codec::imap_types::mailbox::Mailbox as MailboxCodec;
|
||||
use imap_codec::imap_types::response::{Code, CodeOther};
|
||||
use imap_codec::imap_types::search::SearchKey;
|
||||
use imap_codec::imap_types::sequence::SequenceSet;
|
||||
|
||||
use crate::imap::attributes::AttributesProxy;
|
||||
use crate::imap::capability::{ClientCapability, ServerCapability};
|
||||
use crate::imap::command::{anystate, authenticated, MailboxName};
|
||||
use crate::imap::command::authenticated;
|
||||
use crate::imap::flow;
|
||||
use crate::imap::mailbox_view::{MailboxView, UpdateParameters};
|
||||
use crate::imap::response::Response;
|
||||
use crate::mail::user::User;
|
||||
|
||||
pub struct SelectedContext<'a> {
|
||||
pub req: &'a Command<'static>,
|
||||
pub user: &'a Arc<User>,
|
||||
pub mailbox: &'a mut MailboxView,
|
||||
pub server_capabilities: &'a ServerCapability,
|
||||
pub client_capabilities: &'a mut ClientCapability,
|
||||
pub perm: &'a flow::MailboxPerm,
|
||||
}
|
||||
use crate::imap::session::InnerContext;
|
||||
use crate::mail::Mailbox;
|
||||
|
||||
pub async fn dispatch<'a>(
|
||||
ctx: SelectedContext<'a>,
|
||||
) -> Result<(Response<'static>, flow::Transition)> {
|
||||
match &ctx.req.body {
|
||||
// Any State
|
||||
// noop is specific to this state
|
||||
CommandBody::Capability => {
|
||||
anystate::capability(ctx.req.tag.clone(), ctx.server_capabilities)
|
||||
}
|
||||
CommandBody::Logout => anystate::logout(),
|
||||
inner: InnerContext<'a>,
|
||||
user: &'a flow::User,
|
||||
mailbox: &'a Mailbox,
|
||||
) -> Result<(Response, flow::Transition)> {
|
||||
let ctx = StateContext {
|
||||
inner,
|
||||
user,
|
||||
mailbox,
|
||||
};
|
||||
|
||||
// Specific to this state (7 commands + NOOP)
|
||||
CommandBody::Close => match ctx.perm {
|
||||
flow::MailboxPerm::ReadWrite => ctx.close().await,
|
||||
flow::MailboxPerm::ReadOnly => ctx.examine_close().await,
|
||||
},
|
||||
CommandBody::Noop | CommandBody::Check => ctx.noop().await,
|
||||
match &ctx.inner.req.command.body {
|
||||
CommandBody::Fetch {
|
||||
sequence_set,
|
||||
macro_or_item_names,
|
||||
modifiers,
|
||||
attributes,
|
||||
uid,
|
||||
} => {
|
||||
ctx.fetch(sequence_set, macro_or_item_names, modifiers, uid)
|
||||
.await
|
||||
}
|
||||
//@FIXME SearchKey::And is a legacy hack, should be refactored
|
||||
CommandBody::Search {
|
||||
charset,
|
||||
criteria,
|
||||
uid,
|
||||
} => {
|
||||
ctx.search(charset, &SearchKey::And(criteria.clone()), uid)
|
||||
.await
|
||||
}
|
||||
CommandBody::Expunge {
|
||||
// UIDPLUS (rfc4315)
|
||||
uid_sequence_set,
|
||||
} => ctx.expunge(uid_sequence_set).await,
|
||||
CommandBody::Store {
|
||||
sequence_set,
|
||||
kind,
|
||||
response,
|
||||
flags,
|
||||
modifiers,
|
||||
uid,
|
||||
} => {
|
||||
ctx.store(sequence_set, kind, response, flags, modifiers, uid)
|
||||
.await
|
||||
}
|
||||
CommandBody::Copy {
|
||||
sequence_set,
|
||||
mailbox,
|
||||
uid,
|
||||
} => ctx.copy(sequence_set, mailbox, uid).await,
|
||||
CommandBody::Move {
|
||||
sequence_set,
|
||||
mailbox,
|
||||
uid,
|
||||
} => ctx.r#move(sequence_set, mailbox, uid).await,
|
||||
|
||||
// UNSELECT extension (rfc3691)
|
||||
CommandBody::Unselect => ctx.unselect().await,
|
||||
|
||||
// In selected mode, we fallback to authenticated when needed
|
||||
_ => {
|
||||
authenticated::dispatch(authenticated::AuthenticatedContext {
|
||||
req: ctx.req,
|
||||
server_capabilities: ctx.server_capabilities,
|
||||
client_capabilities: ctx.client_capabilities,
|
||||
user: ctx.user,
|
||||
})
|
||||
.await
|
||||
}
|
||||
} => ctx.fetch(sequence_set, attributes, uid).await,
|
||||
_ => authenticated::dispatch(ctx.inner, user).await,
|
||||
}
|
||||
}
|
||||
|
||||
// --- PRIVATE ---
|
||||
|
||||
impl<'a> SelectedContext<'a> {
|
||||
async fn close(self) -> Result<(Response<'static>, flow::Transition)> {
|
||||
// We expunge messages,
|
||||
// but we don't send the untagged EXPUNGE responses
|
||||
let tag = self.req.tag.clone();
|
||||
self.expunge(&None).await?;
|
||||
Ok((
|
||||
Response::build().tag(tag).message("CLOSE completed").ok()?,
|
||||
flow::Transition::Unselect,
|
||||
))
|
||||
}
|
||||
|
||||
/// CLOSE in examined state is not the same as in selected state
|
||||
/// (in selected state it also does an EXPUNGE, here it doesn't)
|
||||
async fn examine_close(self) -> Result<(Response<'static>, flow::Transition)> {
|
||||
Ok((
|
||||
Response::build()
|
||||
.to_req(self.req)
|
||||
.message("CLOSE completed")
|
||||
.ok()?,
|
||||
flow::Transition::Unselect,
|
||||
))
|
||||
}
|
||||
|
||||
async fn unselect(self) -> Result<(Response<'static>, flow::Transition)> {
|
||||
Ok((
|
||||
Response::build()
|
||||
.to_req(self.req)
|
||||
.message("UNSELECT completed")
|
||||
.ok()?,
|
||||
flow::Transition::Unselect,
|
||||
))
|
||||
}
|
||||
struct StateContext<'a> {
|
||||
inner: InnerContext<'a>,
|
||||
user: &'a flow::User,
|
||||
mailbox: &'a Mailbox,
|
||||
}
|
||||
|
||||
impl<'a> StateContext<'a> {
|
||||
pub async fn fetch(
|
||||
self,
|
||||
&self,
|
||||
sequence_set: &SequenceSet,
|
||||
attributes: &'a MacroOrMessageDataItemNames<'static>,
|
||||
modifiers: &[FetchModifier],
|
||||
attributes: &MacroOrFetchAttributes,
|
||||
uid: &bool,
|
||||
) -> Result<(Response<'static>, flow::Transition)> {
|
||||
let ap = AttributesProxy::new(attributes, modifiers, *uid);
|
||||
let mut changed_since: Option<NonZeroU64> = None;
|
||||
modifiers.iter().for_each(|m| match m {
|
||||
FetchModifier::ChangedSince(val) => {
|
||||
changed_since = Some(*val);
|
||||
}
|
||||
});
|
||||
|
||||
match self
|
||||
.mailbox
|
||||
.fetch(sequence_set, &ap, changed_since, uid)
|
||||
.await
|
||||
{
|
||||
Ok(resp) => {
|
||||
// Capabilities enabling logic only on successful command
|
||||
// (according to my understanding of the spec)
|
||||
self.client_capabilities.attributes_enable(&ap);
|
||||
self.client_capabilities.fetch_modifiers_enable(modifiers);
|
||||
|
||||
// Response to the client
|
||||
Ok((
|
||||
Response::build()
|
||||
.to_req(self.req)
|
||||
.message("FETCH completed")
|
||||
.set_body(resp)
|
||||
.ok()?,
|
||||
flow::Transition::None,
|
||||
))
|
||||
}
|
||||
Err(e) => Ok((
|
||||
Response::build()
|
||||
.to_req(self.req)
|
||||
.message(e.to_string())
|
||||
.no()?,
|
||||
flow::Transition::None,
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn search(
|
||||
self,
|
||||
charset: &Option<Charset<'a>>,
|
||||
criteria: &SearchKey<'a>,
|
||||
uid: &bool,
|
||||
) -> Result<(Response<'static>, flow::Transition)> {
|
||||
let (found, enable_condstore) = self.mailbox.search(charset, criteria, *uid).await?;
|
||||
if enable_condstore {
|
||||
self.client_capabilities.enable_condstore();
|
||||
}
|
||||
Ok((
|
||||
Response::build()
|
||||
.to_req(self.req)
|
||||
.set_body(found)
|
||||
.message("SEARCH completed")
|
||||
.ok()?,
|
||||
flow::Transition::None,
|
||||
))
|
||||
}
|
||||
|
||||
pub async fn noop(self) -> Result<(Response<'static>, flow::Transition)> {
|
||||
self.mailbox.internal.mailbox.force_sync().await?;
|
||||
|
||||
let updates = self.mailbox.update(UpdateParameters::default()).await?;
|
||||
Ok((
|
||||
Response::build()
|
||||
.to_req(self.req)
|
||||
.message("NOOP completed.")
|
||||
.set_body(updates)
|
||||
.ok()?,
|
||||
flow::Transition::None,
|
||||
))
|
||||
}
|
||||
|
||||
async fn expunge(
|
||||
self,
|
||||
uid_sequence_set: &Option<SequenceSet>,
|
||||
) -> Result<(Response<'static>, flow::Transition)> {
|
||||
if let Some(failed) = self.fail_read_only() {
|
||||
return Ok((failed, flow::Transition::None));
|
||||
}
|
||||
|
||||
let tag = self.req.tag.clone();
|
||||
let data = self.mailbox.expunge(uid_sequence_set).await?;
|
||||
|
||||
Ok((
|
||||
Response::build()
|
||||
.tag(tag)
|
||||
.message("EXPUNGE completed")
|
||||
.set_body(data)
|
||||
.ok()?,
|
||||
flow::Transition::None,
|
||||
))
|
||||
}
|
||||
|
||||
async fn store(
|
||||
self,
|
||||
sequence_set: &SequenceSet,
|
||||
kind: &StoreType,
|
||||
response: &StoreResponse,
|
||||
flags: &[Flag<'a>],
|
||||
modifiers: &[StoreModifier],
|
||||
uid: &bool,
|
||||
) -> Result<(Response<'static>, flow::Transition)> {
|
||||
if let Some(failed) = self.fail_read_only() {
|
||||
return Ok((failed, flow::Transition::None));
|
||||
}
|
||||
|
||||
let mut unchanged_since: Option<NonZeroU64> = None;
|
||||
modifiers.iter().for_each(|m| match m {
|
||||
StoreModifier::UnchangedSince(val) => {
|
||||
unchanged_since = Some(*val);
|
||||
}
|
||||
});
|
||||
|
||||
let (data, modified) = self
|
||||
.mailbox
|
||||
.store(sequence_set, kind, response, flags, unchanged_since, uid)
|
||||
.await?;
|
||||
|
||||
let mut ok_resp = Response::build()
|
||||
.to_req(self.req)
|
||||
.message("STORE completed")
|
||||
.set_body(data);
|
||||
|
||||
match modified[..] {
|
||||
[] => (),
|
||||
[_head, ..] => {
|
||||
let modified_str = format!(
|
||||
"MODIFIED {}",
|
||||
modified
|
||||
.into_iter()
|
||||
.map(|x| x.to_string())
|
||||
.collect::<Vec<_>>()
|
||||
.join(",")
|
||||
);
|
||||
ok_resp = ok_resp.code(Code::Other(CodeOther::unvalidated(
|
||||
modified_str.into_bytes(),
|
||||
)));
|
||||
}
|
||||
};
|
||||
|
||||
self.client_capabilities.store_modifiers_enable(modifiers);
|
||||
|
||||
Ok((ok_resp.ok()?, flow::Transition::None))
|
||||
}
|
||||
|
||||
async fn copy(
|
||||
self,
|
||||
sequence_set: &SequenceSet,
|
||||
mailbox: &MailboxCodec<'a>,
|
||||
uid: &bool,
|
||||
) -> Result<(Response<'static>, flow::Transition)> {
|
||||
//@FIXME Could copy be valid in EXAMINE mode?
|
||||
if let Some(failed) = self.fail_read_only() {
|
||||
return Ok((failed, flow::Transition::None));
|
||||
}
|
||||
|
||||
let name: &str = MailboxName(mailbox).try_into()?;
|
||||
|
||||
let mb_opt = self.user.open_mailbox(&name).await?;
|
||||
let mb = match mb_opt {
|
||||
Some(mb) => mb,
|
||||
None => {
|
||||
return Ok((
|
||||
Response::build()
|
||||
.to_req(self.req)
|
||||
.message("Destination mailbox does not exist")
|
||||
.code(Code::TryCreate)
|
||||
.no()?,
|
||||
flow::Transition::None,
|
||||
))
|
||||
}
|
||||
};
|
||||
|
||||
let (uidval, uid_map) = self.mailbox.copy(sequence_set, mb, uid).await?;
|
||||
|
||||
let copyuid_str = format!(
|
||||
"{} {} {}",
|
||||
uidval,
|
||||
uid_map
|
||||
.iter()
|
||||
.map(|(sid, _)| format!("{}", sid))
|
||||
.collect::<Vec<_>>()
|
||||
.join(","),
|
||||
uid_map
|
||||
.iter()
|
||||
.map(|(_, tuid)| format!("{}", tuid))
|
||||
.collect::<Vec<_>>()
|
||||
.join(",")
|
||||
);
|
||||
|
||||
Ok((
|
||||
Response::build()
|
||||
.to_req(self.req)
|
||||
.message("COPY completed")
|
||||
.code(Code::Other(CodeOther::unvalidated(
|
||||
format!("COPYUID {}", copyuid_str).into_bytes(),
|
||||
)))
|
||||
.ok()?,
|
||||
flow::Transition::None,
|
||||
))
|
||||
}
|
||||
|
||||
async fn r#move(
|
||||
self,
|
||||
sequence_set: &SequenceSet,
|
||||
mailbox: &MailboxCodec<'a>,
|
||||
uid: &bool,
|
||||
) -> Result<(Response<'static>, flow::Transition)> {
|
||||
if let Some(failed) = self.fail_read_only() {
|
||||
return Ok((failed, flow::Transition::None));
|
||||
}
|
||||
|
||||
let name: &str = MailboxName(mailbox).try_into()?;
|
||||
|
||||
let mb_opt = self.user.open_mailbox(&name).await?;
|
||||
let mb = match mb_opt {
|
||||
Some(mb) => mb,
|
||||
None => {
|
||||
return Ok((
|
||||
Response::build()
|
||||
.to_req(self.req)
|
||||
.message("Destination mailbox does not exist")
|
||||
.code(Code::TryCreate)
|
||||
.no()?,
|
||||
flow::Transition::None,
|
||||
))
|
||||
}
|
||||
};
|
||||
|
||||
let (uidval, uid_map, data) = self.mailbox.r#move(sequence_set, mb, uid).await?;
|
||||
|
||||
// compute code
|
||||
let copyuid_str = format!(
|
||||
"{} {} {}",
|
||||
uidval,
|
||||
uid_map
|
||||
.iter()
|
||||
.map(|(sid, _)| format!("{}", sid))
|
||||
.collect::<Vec<_>>()
|
||||
.join(","),
|
||||
uid_map
|
||||
.iter()
|
||||
.map(|(_, tuid)| format!("{}", tuid))
|
||||
.collect::<Vec<_>>()
|
||||
.join(",")
|
||||
);
|
||||
|
||||
Ok((
|
||||
Response::build()
|
||||
.to_req(self.req)
|
||||
.message("COPY completed")
|
||||
.code(Code::Other(CodeOther::unvalidated(
|
||||
format!("COPYUID {}", copyuid_str).into_bytes(),
|
||||
)))
|
||||
.set_body(data)
|
||||
.ok()?,
|
||||
flow::Transition::None,
|
||||
))
|
||||
}
|
||||
|
||||
fn fail_read_only(&self) -> Option<Response<'static>> {
|
||||
match self.perm {
|
||||
flow::MailboxPerm::ReadWrite => None,
|
||||
flow::MailboxPerm::ReadOnly => Some(
|
||||
Response::build()
|
||||
.to_req(self.req)
|
||||
.message("Write command are forbidden while exmining mailbox")
|
||||
.no()
|
||||
.unwrap(),
|
||||
),
|
||||
}
|
||||
) -> Result<(Response, flow::Transition)> {
|
||||
Ok((Response::bad("Not implemented")?, flow::Transition::No))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,30 +0,0 @@
|
|||
use imap_codec::imap_types::core::Atom;
|
||||
use imap_codec::imap_types::flag::{Flag, FlagFetch};
|
||||
|
||||
pub fn from_str(f: &str) -> Option<FlagFetch<'static>> {
|
||||
match f.chars().next() {
|
||||
Some('\\') => match f {
|
||||
"\\Seen" => Some(FlagFetch::Flag(Flag::Seen)),
|
||||
"\\Answered" => Some(FlagFetch::Flag(Flag::Answered)),
|
||||
"\\Flagged" => Some(FlagFetch::Flag(Flag::Flagged)),
|
||||
"\\Deleted" => Some(FlagFetch::Flag(Flag::Deleted)),
|
||||
"\\Draft" => Some(FlagFetch::Flag(Flag::Draft)),
|
||||
"\\Recent" => Some(FlagFetch::Recent),
|
||||
_ => match Atom::try_from(f.strip_prefix('\\').unwrap().to_string()) {
|
||||
Err(_) => {
|
||||
tracing::error!(flag=%f, "Unable to encode flag as IMAP atom");
|
||||
None
|
||||
}
|
||||
Ok(a) => Some(FlagFetch::Flag(Flag::system(a))),
|
||||
},
|
||||
},
|
||||
Some(_) => match Atom::try_from(f.to_string()) {
|
||||
Err(_) => {
|
||||
tracing::error!(flag=%f, "Unable to encode flag as IMAP atom");
|
||||
None
|
||||
}
|
||||
Ok(a) => Some(FlagFetch::Flag(Flag::keyword(a))),
|
||||
},
|
||||
None => None,
|
||||
}
|
||||
}
|
||||
103
src/imap/flow.rs
103
src/imap/flow.rs
|
|
@ -1,12 +1,13 @@
|
|||
use std::error::Error as StdError;
|
||||
use std::fmt;
|
||||
use std::sync::Arc;
|
||||
|
||||
use imap_codec::imap_types::core::Tag;
|
||||
use tokio::sync::Notify;
|
||||
use crate::login::Credentials;
|
||||
use crate::mail::Mailbox;
|
||||
|
||||
use crate::imap::mailbox_view::MailboxView;
|
||||
use crate::mail::user::User;
|
||||
pub struct User {
|
||||
pub name: String,
|
||||
pub creds: Credentials,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum Error {
|
||||
|
|
@ -21,94 +22,30 @@ impl StdError for Error {}
|
|||
|
||||
pub enum State {
|
||||
NotAuthenticated,
|
||||
Authenticated(Arc<User>),
|
||||
Selected(Arc<User>, MailboxView, MailboxPerm),
|
||||
Idle(
|
||||
Arc<User>,
|
||||
MailboxView,
|
||||
MailboxPerm,
|
||||
Tag<'static>,
|
||||
Arc<Notify>,
|
||||
),
|
||||
Authenticated(User),
|
||||
Selected(User, Mailbox),
|
||||
Logout,
|
||||
}
|
||||
impl State {
|
||||
pub fn notify(&self) -> Option<Arc<Notify>> {
|
||||
match self {
|
||||
Self::Idle(_, _, _, _, anotif) => Some(anotif.clone()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
impl fmt::Display for State {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
use State::*;
|
||||
match self {
|
||||
NotAuthenticated => write!(f, "NotAuthenticated"),
|
||||
Authenticated(..) => write!(f, "Authenticated"),
|
||||
Selected(..) => write!(f, "Selected"),
|
||||
Idle(..) => write!(f, "Idle"),
|
||||
Logout => write!(f, "Logout"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub enum MailboxPerm {
|
||||
ReadOnly,
|
||||
ReadWrite,
|
||||
}
|
||||
|
||||
pub enum Transition {
|
||||
None,
|
||||
Authenticate(Arc<User>),
|
||||
Select(MailboxView, MailboxPerm),
|
||||
Idle(Tag<'static>, Notify),
|
||||
UnIdle,
|
||||
No,
|
||||
Authenticate(User),
|
||||
Select(Mailbox),
|
||||
Unselect,
|
||||
Logout,
|
||||
}
|
||||
impl fmt::Display for Transition {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
use Transition::*;
|
||||
match self {
|
||||
None => write!(f, "None"),
|
||||
Authenticate(..) => write!(f, "Authenticated"),
|
||||
Select(..) => write!(f, "Selected"),
|
||||
Idle(..) => write!(f, "Idle"),
|
||||
UnIdle => write!(f, "UnIdle"),
|
||||
Unselect => write!(f, "Unselect"),
|
||||
Logout => write!(f, "Logout"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// See RFC3501 section 3.
|
||||
// https://datatracker.ietf.org/doc/html/rfc3501#page-13
|
||||
impl State {
|
||||
pub fn apply(&mut self, tr: Transition) -> Result<(), Error> {
|
||||
tracing::debug!(state=%self, transition=%tr, "try change state");
|
||||
|
||||
let new_state = match (std::mem::replace(self, State::Logout), tr) {
|
||||
(s, Transition::None) => s,
|
||||
(State::NotAuthenticated, Transition::Authenticate(u)) => State::Authenticated(u),
|
||||
(State::Authenticated(u) | State::Selected(u, _, _), Transition::Select(m, p)) => {
|
||||
State::Selected(u, m, p)
|
||||
}
|
||||
(State::Selected(u, _, _), Transition::Unselect) => State::Authenticated(u.clone()),
|
||||
(State::Selected(u, m, p), Transition::Idle(t, s)) => {
|
||||
State::Idle(u, m, p, t, Arc::new(s))
|
||||
}
|
||||
(State::Idle(u, m, p, _, _), Transition::UnIdle) => State::Selected(u, m, p),
|
||||
(_, Transition::Logout) => State::Logout,
|
||||
(s, t) => {
|
||||
tracing::error!(state=%s, transition=%t, "forbidden transition");
|
||||
return Err(Error::ForbiddenTransition);
|
||||
}
|
||||
};
|
||||
*self = new_state;
|
||||
tracing::debug!(state=%self, "transition succeeded");
|
||||
|
||||
Ok(())
|
||||
pub fn apply(self, tr: Transition) -> Result<Self, Error> {
|
||||
match (self, tr) {
|
||||
(s, Transition::No) => Ok(s),
|
||||
(State::NotAuthenticated, Transition::Authenticate(u)) => Ok(State::Authenticated(u)),
|
||||
(State::Authenticated(u), Transition::Select(m)) => Ok(State::Selected(u, m)),
|
||||
(State::Selected(u, _), Transition::Unselect) => Ok(State::Authenticated(u)),
|
||||
(_, Transition::Logout) => Ok(State::Logout),
|
||||
_ => Err(Error::ForbiddenTransition),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,109 +0,0 @@
|
|||
use anyhow::{anyhow, Result};
|
||||
use chrono::naive::NaiveDate;
|
||||
|
||||
use imap_codec::imap_types::core::{IString, NString};
|
||||
use imap_codec::imap_types::envelope::{Address, Envelope};
|
||||
|
||||
use eml_codec::imf;
|
||||
|
||||
pub struct ImfView<'a>(pub &'a imf::Imf<'a>);
|
||||
|
||||
impl<'a> ImfView<'a> {
|
||||
pub fn naive_date(&self) -> Result<NaiveDate> {
|
||||
Ok(self.0.date.ok_or(anyhow!("date is not set"))?.date_naive())
|
||||
}
|
||||
|
||||
/// Envelope rules are defined in RFC 3501, section 7.4.2
|
||||
/// https://datatracker.ietf.org/doc/html/rfc3501#section-7.4.2
|
||||
///
|
||||
/// Some important notes:
|
||||
///
|
||||
/// If the Sender or Reply-To lines are absent in the [RFC-2822]
|
||||
/// header, or are present but empty, the server sets the
|
||||
/// corresponding member of the envelope to be the same value as
|
||||
/// the from member (the client is not expected to know to do
|
||||
/// this). Note: [RFC-2822] requires that all messages have a valid
|
||||
/// From header. Therefore, the from, sender, and reply-to
|
||||
/// members in the envelope can not be NIL.
|
||||
///
|
||||
/// If the Date, Subject, In-Reply-To, and Message-ID header lines
|
||||
/// are absent in the [RFC-2822] header, the corresponding member
|
||||
/// of the envelope is NIL; if these header lines are present but
|
||||
/// empty the corresponding member of the envelope is the empty
|
||||
/// string.
|
||||
|
||||
//@FIXME return an error if the envelope is invalid instead of panicking
|
||||
//@FIXME some fields must be defaulted if there are not set.
|
||||
pub fn message_envelope(&self) -> Envelope<'static> {
|
||||
let msg = self.0;
|
||||
let from = msg.from.iter().map(convert_mbx).collect::<Vec<_>>();
|
||||
|
||||
Envelope {
|
||||
date: NString(
|
||||
msg.date
|
||||
.as_ref()
|
||||
.map(|d| IString::try_from(d.to_rfc3339()).unwrap()),
|
||||
),
|
||||
subject: NString(
|
||||
msg.subject
|
||||
.as_ref()
|
||||
.map(|d| IString::try_from(d.to_string()).unwrap()),
|
||||
),
|
||||
sender: msg
|
||||
.sender
|
||||
.as_ref()
|
||||
.map(|v| vec![convert_mbx(v)])
|
||||
.unwrap_or(from.clone()),
|
||||
reply_to: if msg.reply_to.is_empty() {
|
||||
from.clone()
|
||||
} else {
|
||||
convert_addresses(&msg.reply_to)
|
||||
},
|
||||
from,
|
||||
to: convert_addresses(&msg.to),
|
||||
cc: convert_addresses(&msg.cc),
|
||||
bcc: convert_addresses(&msg.bcc),
|
||||
in_reply_to: NString(
|
||||
msg.in_reply_to
|
||||
.iter()
|
||||
.next()
|
||||
.map(|d| IString::try_from(d.to_string()).unwrap()),
|
||||
),
|
||||
message_id: NString(
|
||||
msg.msg_id
|
||||
.as_ref()
|
||||
.map(|d| IString::try_from(d.to_string()).unwrap()),
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn convert_addresses(addrlist: &Vec<imf::address::AddressRef>) -> Vec<Address<'static>> {
|
||||
let mut acc = vec![];
|
||||
for item in addrlist {
|
||||
match item {
|
||||
imf::address::AddressRef::Single(a) => acc.push(convert_mbx(a)),
|
||||
imf::address::AddressRef::Many(l) => acc.extend(l.participants.iter().map(convert_mbx)),
|
||||
}
|
||||
}
|
||||
return acc;
|
||||
}
|
||||
|
||||
pub fn convert_mbx(addr: &imf::mailbox::MailboxRef) -> Address<'static> {
|
||||
Address {
|
||||
name: NString(
|
||||
addr.name
|
||||
.as_ref()
|
||||
.map(|x| IString::try_from(x.to_string()).unwrap()),
|
||||
),
|
||||
// SMTP at-domain-list (source route) seems obsolete since at least 1991
|
||||
// https://www.mhonarc.org/archive/html/ietf-822/1991-06/msg00060.html
|
||||
adl: NString(None),
|
||||
mailbox: NString(Some(
|
||||
IString::try_from(addr.addrspec.local_part.to_string()).unwrap(),
|
||||
)),
|
||||
host: NString(Some(
|
||||
IString::try_from(addr.addrspec.domain.to_string()).unwrap(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
|
@ -1,211 +0,0 @@
|
|||
use std::num::{NonZeroU32, NonZeroU64};
|
||||
|
||||
use anyhow::{anyhow, Result};
|
||||
use imap_codec::imap_types::sequence::{SeqOrUid, Sequence, SequenceSet};
|
||||
|
||||
use crate::mail::uidindex::{ImapUid, ModSeq, UidIndex};
|
||||
use crate::mail::unique_ident::UniqueIdent;
|
||||
|
||||
pub struct Index<'a> {
|
||||
pub imap_index: Vec<MailIndex<'a>>,
|
||||
pub internal: &'a UidIndex,
|
||||
}
|
||||
impl<'a> Index<'a> {
|
||||
pub fn new(internal: &'a UidIndex) -> Result<Self> {
|
||||
let imap_index = internal
|
||||
.idx_by_uid
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i_enum, (&uid, &uuid))| {
|
||||
let (_, modseq, flags) = internal
|
||||
.table
|
||||
.get(&uuid)
|
||||
.ok_or(anyhow!("mail is missing from index"))?;
|
||||
let i_int: u32 = (i_enum + 1).try_into()?;
|
||||
let i: NonZeroU32 = i_int.try_into()?;
|
||||
|
||||
Ok(MailIndex {
|
||||
i,
|
||||
uid,
|
||||
uuid,
|
||||
modseq: *modseq,
|
||||
flags,
|
||||
})
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
|
||||
Ok(Self {
|
||||
imap_index,
|
||||
internal,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn last(&'a self) -> Option<&'a MailIndex<'a>> {
|
||||
self.imap_index.last()
|
||||
}
|
||||
|
||||
/// Fetch mail descriptors based on a sequence of UID
|
||||
///
|
||||
/// Complexity analysis:
|
||||
/// - Sort is O(n * log n) where n is the number of uid generated by the sequence
|
||||
/// - Finding the starting point in the index O(log m) where m is the size of the mailbox
|
||||
/// While n =< m, it's not clear if the difference is big or not.
|
||||
///
|
||||
/// For now, the algorithm tries to be fast for small values of n,
|
||||
/// as it is what is expected by clients.
|
||||
///
|
||||
/// So we assume for our implementation that : n << m.
|
||||
/// It's not true for full mailbox searches for example...
|
||||
pub fn fetch_on_uid(&'a self, sequence_set: &SequenceSet) -> Vec<&'a MailIndex<'a>> {
|
||||
if self.imap_index.is_empty() {
|
||||
return vec![];
|
||||
}
|
||||
let largest = self.last().expect("The mailbox is not empty").uid;
|
||||
let mut unroll_seq = sequence_set.iter(largest).collect::<Vec<_>>();
|
||||
unroll_seq.sort();
|
||||
|
||||
let start_seq = match unroll_seq.iter().next() {
|
||||
Some(elem) => elem,
|
||||
None => return vec![],
|
||||
};
|
||||
|
||||
// Quickly jump to the right point in the mailbox vector O(log m) instead
|
||||
// of iterating one by one O(m). Works only because both unroll_seq & imap_index are sorted per uid.
|
||||
let mut imap_idx = {
|
||||
let start_idx = self
|
||||
.imap_index
|
||||
.partition_point(|mail_idx| &mail_idx.uid < start_seq);
|
||||
&self.imap_index[start_idx..]
|
||||
};
|
||||
|
||||
let mut acc = vec![];
|
||||
for wanted_uid in unroll_seq.iter() {
|
||||
// Slide the window forward as long as its first element is lower than our wanted uid.
|
||||
let start_idx = match imap_idx.iter().position(|midx| &midx.uid >= wanted_uid) {
|
||||
Some(v) => v,
|
||||
None => break,
|
||||
};
|
||||
imap_idx = &imap_idx[start_idx..];
|
||||
|
||||
// If the beginning of our new window is the uid we want, we collect it
|
||||
if &imap_idx[0].uid == wanted_uid {
|
||||
acc.push(&imap_idx[0]);
|
||||
}
|
||||
}
|
||||
|
||||
acc
|
||||
}
|
||||
|
||||
pub fn fetch_on_id(&'a self, sequence_set: &SequenceSet) -> Result<Vec<&'a MailIndex<'a>>> {
|
||||
if self.imap_index.is_empty() {
|
||||
return Ok(vec![]);
|
||||
}
|
||||
let largest = NonZeroU32::try_from(self.imap_index.len() as u32)?;
|
||||
let mut acc = sequence_set
|
||||
.iter(largest)
|
||||
.map(|wanted_id| {
|
||||
self.imap_index
|
||||
.get((wanted_id.get() as usize) - 1)
|
||||
.ok_or(anyhow!("Mail not found"))
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
|
||||
// Sort the result to be consistent with UID
|
||||
acc.sort_by(|a, b| a.i.cmp(&b.i));
|
||||
|
||||
Ok(acc)
|
||||
}
|
||||
|
||||
pub fn fetch(
|
||||
self: &'a Index<'a>,
|
||||
sequence_set: &SequenceSet,
|
||||
by_uid: bool,
|
||||
) -> Result<Vec<&'a MailIndex<'a>>> {
|
||||
match by_uid {
|
||||
true => Ok(self.fetch_on_uid(sequence_set)),
|
||||
_ => self.fetch_on_id(sequence_set),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn fetch_changed_since(
|
||||
self: &'a Index<'a>,
|
||||
sequence_set: &SequenceSet,
|
||||
maybe_modseq: Option<NonZeroU64>,
|
||||
by_uid: bool,
|
||||
) -> Result<Vec<&'a MailIndex<'a>>> {
|
||||
let raw = self.fetch(sequence_set, by_uid)?;
|
||||
let res = match maybe_modseq {
|
||||
Some(pit) => raw.into_iter().filter(|midx| midx.modseq > pit).collect(),
|
||||
None => raw,
|
||||
};
|
||||
|
||||
Ok(res)
|
||||
}
|
||||
|
||||
pub fn fetch_unchanged_since(
|
||||
self: &'a Index<'a>,
|
||||
sequence_set: &SequenceSet,
|
||||
maybe_modseq: Option<NonZeroU64>,
|
||||
by_uid: bool,
|
||||
) -> Result<(Vec<&'a MailIndex<'a>>, Vec<&'a MailIndex<'a>>)> {
|
||||
let raw = self.fetch(sequence_set, by_uid)?;
|
||||
let res = match maybe_modseq {
|
||||
Some(pit) => raw.into_iter().partition(|midx| midx.modseq <= pit),
|
||||
None => (raw, vec![]),
|
||||
};
|
||||
|
||||
Ok(res)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct MailIndex<'a> {
|
||||
pub i: NonZeroU32,
|
||||
pub uid: ImapUid,
|
||||
pub uuid: UniqueIdent,
|
||||
pub modseq: ModSeq,
|
||||
pub flags: &'a Vec<String>,
|
||||
}
|
||||
|
||||
impl<'a> MailIndex<'a> {
|
||||
// The following functions are used to implement the SEARCH command
|
||||
pub fn is_in_sequence_i(&self, seq: &Sequence) -> bool {
|
||||
match seq {
|
||||
Sequence::Single(SeqOrUid::Asterisk) => true,
|
||||
Sequence::Single(SeqOrUid::Value(target)) => target == &self.i,
|
||||
Sequence::Range(SeqOrUid::Asterisk, SeqOrUid::Value(x))
|
||||
| Sequence::Range(SeqOrUid::Value(x), SeqOrUid::Asterisk) => x <= &self.i,
|
||||
Sequence::Range(SeqOrUid::Value(x1), SeqOrUid::Value(x2)) => {
|
||||
if x1 < x2 {
|
||||
x1 <= &self.i && &self.i <= x2
|
||||
} else {
|
||||
x1 >= &self.i && &self.i >= x2
|
||||
}
|
||||
}
|
||||
Sequence::Range(SeqOrUid::Asterisk, SeqOrUid::Asterisk) => true,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_in_sequence_uid(&self, seq: &Sequence) -> bool {
|
||||
match seq {
|
||||
Sequence::Single(SeqOrUid::Asterisk) => true,
|
||||
Sequence::Single(SeqOrUid::Value(target)) => target == &self.uid,
|
||||
Sequence::Range(SeqOrUid::Asterisk, SeqOrUid::Value(x))
|
||||
| Sequence::Range(SeqOrUid::Value(x), SeqOrUid::Asterisk) => x <= &self.uid,
|
||||
Sequence::Range(SeqOrUid::Value(x1), SeqOrUid::Value(x2)) => {
|
||||
if x1 < x2 {
|
||||
x1 <= &self.uid && &self.uid <= x2
|
||||
} else {
|
||||
x1 >= &self.uid && &self.uid >= x2
|
||||
}
|
||||
}
|
||||
Sequence::Range(SeqOrUid::Asterisk, SeqOrUid::Asterisk) => true,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_flag_set(&self, flag: &str) -> bool {
|
||||
self.flags
|
||||
.iter()
|
||||
.any(|candidate| candidate.as_str() == flag)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,306 +0,0 @@
|
|||
use std::num::NonZeroU32;
|
||||
|
||||
use anyhow::{anyhow, bail, Result};
|
||||
use chrono::{naive::NaiveDate, DateTime as ChronoDateTime, Local, Offset, TimeZone, Utc};
|
||||
|
||||
use imap_codec::imap_types::core::NString;
|
||||
use imap_codec::imap_types::datetime::DateTime;
|
||||
use imap_codec::imap_types::fetch::{
|
||||
MessageDataItem, MessageDataItemName, Section as FetchSection,
|
||||
};
|
||||
use imap_codec::imap_types::flag::Flag;
|
||||
use imap_codec::imap_types::response::Data;
|
||||
|
||||
use eml_codec::{
|
||||
imf,
|
||||
part::{composite::Message, AnyPart},
|
||||
};
|
||||
|
||||
use crate::mail::query::QueryResult;
|
||||
|
||||
use crate::imap::attributes::AttributesProxy;
|
||||
use crate::imap::flags;
|
||||
use crate::imap::imf_view::ImfView;
|
||||
use crate::imap::index::MailIndex;
|
||||
use crate::imap::mime_view;
|
||||
use crate::imap::response::Body;
|
||||
|
||||
pub struct MailView<'a> {
|
||||
pub in_idx: &'a MailIndex<'a>,
|
||||
pub query_result: &'a QueryResult,
|
||||
pub content: FetchedMail<'a>,
|
||||
}
|
||||
|
||||
impl<'a> MailView<'a> {
|
||||
pub fn new(query_result: &'a QueryResult, in_idx: &'a MailIndex<'a>) -> Result<MailView<'a>> {
|
||||
Ok(Self {
|
||||
in_idx,
|
||||
query_result,
|
||||
content: match query_result {
|
||||
QueryResult::FullResult { content, .. } => {
|
||||
let (_, parsed) =
|
||||
eml_codec::parse_message(&content).or(Err(anyhow!("Invalid mail body")))?;
|
||||
FetchedMail::full_from_message(parsed)
|
||||
}
|
||||
QueryResult::PartialResult { metadata, .. } => {
|
||||
let (_, parsed) = eml_codec::parse_message(&metadata.headers)
|
||||
.or(Err(anyhow!("unable to parse email headers")))?;
|
||||
FetchedMail::partial_from_message(parsed)
|
||||
}
|
||||
QueryResult::IndexResult { .. } => FetchedMail::IndexOnly,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
pub fn imf(&self) -> Option<ImfView> {
|
||||
self.content.as_imf().map(ImfView)
|
||||
}
|
||||
|
||||
pub fn selected_mime(&'a self) -> Option<mime_view::SelectedMime<'a>> {
|
||||
self.content.as_anypart().ok().map(mime_view::SelectedMime)
|
||||
}
|
||||
|
||||
pub fn filter(&self, ap: &AttributesProxy) -> Result<(Body<'static>, SeenFlag)> {
|
||||
let mut seen = SeenFlag::DoNothing;
|
||||
let res_attrs = ap
|
||||
.attrs
|
||||
.iter()
|
||||
.map(|attr| match attr {
|
||||
MessageDataItemName::Uid => Ok(self.uid()),
|
||||
MessageDataItemName::Flags => Ok(self.flags()),
|
||||
MessageDataItemName::Rfc822Size => self.rfc_822_size(),
|
||||
MessageDataItemName::Rfc822Header => self.rfc_822_header(),
|
||||
MessageDataItemName::Rfc822Text => self.rfc_822_text(),
|
||||
MessageDataItemName::Rfc822 => {
|
||||
if self.is_not_yet_seen() {
|
||||
seen = SeenFlag::MustAdd;
|
||||
}
|
||||
self.rfc822()
|
||||
}
|
||||
MessageDataItemName::Envelope => Ok(self.envelope()),
|
||||
MessageDataItemName::Body => self.body(),
|
||||
MessageDataItemName::BodyStructure => self.body_structure(),
|
||||
MessageDataItemName::BodyExt {
|
||||
section,
|
||||
partial,
|
||||
peek,
|
||||
} => {
|
||||
let (body, has_seen) = self.body_ext(section, partial, peek)?;
|
||||
seen = has_seen;
|
||||
Ok(body)
|
||||
}
|
||||
MessageDataItemName::InternalDate => self.internal_date(),
|
||||
MessageDataItemName::ModSeq => Ok(self.modseq()),
|
||||
})
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
|
||||
Ok((
|
||||
Body::Data(Data::Fetch {
|
||||
seq: self.in_idx.i,
|
||||
items: res_attrs.try_into()?,
|
||||
}),
|
||||
seen,
|
||||
))
|
||||
}
|
||||
|
||||
pub fn stored_naive_date(&self) -> Result<NaiveDate> {
|
||||
let mail_meta = self.query_result.metadata().expect("metadata were fetched");
|
||||
let mail_ts: i64 = mail_meta.internaldate.try_into()?;
|
||||
let msg_date: ChronoDateTime<Local> = ChronoDateTime::from_timestamp(mail_ts, 0)
|
||||
.ok_or(anyhow!("unable to parse timestamp"))?
|
||||
.with_timezone(&Local);
|
||||
|
||||
Ok(msg_date.date_naive())
|
||||
}
|
||||
|
||||
pub fn is_header_contains_pattern(&self, hdr: &[u8], pattern: &[u8]) -> bool {
|
||||
let mime = match self.selected_mime() {
|
||||
None => return false,
|
||||
Some(x) => x,
|
||||
};
|
||||
|
||||
let val = match mime.header_value(hdr) {
|
||||
None => return false,
|
||||
Some(x) => x,
|
||||
};
|
||||
|
||||
val.windows(pattern.len()).any(|win| win == pattern)
|
||||
}
|
||||
|
||||
// Private function, mainly for filter!
|
||||
fn uid(&self) -> MessageDataItem<'static> {
|
||||
MessageDataItem::Uid(self.in_idx.uid.clone())
|
||||
}
|
||||
|
||||
fn flags(&self) -> MessageDataItem<'static> {
|
||||
MessageDataItem::Flags(
|
||||
self.in_idx
|
||||
.flags
|
||||
.iter()
|
||||
.filter_map(|f| flags::from_str(f))
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
|
||||
fn rfc_822_size(&self) -> Result<MessageDataItem<'static>> {
|
||||
let sz = self
|
||||
.query_result
|
||||
.metadata()
|
||||
.ok_or(anyhow!("mail metadata are required"))?
|
||||
.rfc822_size;
|
||||
Ok(MessageDataItem::Rfc822Size(sz as u32))
|
||||
}
|
||||
|
||||
fn rfc_822_header(&self) -> Result<MessageDataItem<'static>> {
|
||||
let hdrs: NString = self
|
||||
.query_result
|
||||
.metadata()
|
||||
.ok_or(anyhow!("mail metadata are required"))?
|
||||
.headers
|
||||
.to_vec()
|
||||
.try_into()?;
|
||||
Ok(MessageDataItem::Rfc822Header(hdrs))
|
||||
}
|
||||
|
||||
fn rfc_822_text(&self) -> Result<MessageDataItem<'static>> {
|
||||
let txt: NString = self.content.as_msg()?.raw_body.to_vec().try_into()?;
|
||||
Ok(MessageDataItem::Rfc822Text(txt))
|
||||
}
|
||||
|
||||
fn rfc822(&self) -> Result<MessageDataItem<'static>> {
|
||||
let full: NString = self.content.as_msg()?.raw_part.to_vec().try_into()?;
|
||||
Ok(MessageDataItem::Rfc822(full))
|
||||
}
|
||||
|
||||
fn envelope(&self) -> MessageDataItem<'static> {
|
||||
MessageDataItem::Envelope(
|
||||
self.imf()
|
||||
.expect("an imf object is derivable from fetchedmail")
|
||||
.message_envelope(),
|
||||
)
|
||||
}
|
||||
|
||||
fn body(&self) -> Result<MessageDataItem<'static>> {
|
||||
Ok(MessageDataItem::Body(mime_view::bodystructure(
|
||||
self.content.as_msg()?.child.as_ref(),
|
||||
false,
|
||||
)?))
|
||||
}
|
||||
|
||||
fn body_structure(&self) -> Result<MessageDataItem<'static>> {
|
||||
Ok(MessageDataItem::BodyStructure(mime_view::bodystructure(
|
||||
self.content.as_msg()?.child.as_ref(),
|
||||
true,
|
||||
)?))
|
||||
}
|
||||
|
||||
fn is_not_yet_seen(&self) -> bool {
|
||||
let seen_flag = Flag::Seen.to_string();
|
||||
!self.in_idx.flags.iter().any(|x| *x == seen_flag)
|
||||
}
|
||||
|
||||
/// maps to BODY[<section>]<<partial>> and BODY.PEEK[<section>]<<partial>>
|
||||
/// peek does not implicitly set the \Seen flag
|
||||
/// eg. BODY[HEADER.FIELDS (DATE FROM)]
|
||||
/// eg. BODY[]<0.2048>
|
||||
fn body_ext(
|
||||
&self,
|
||||
section: &Option<FetchSection<'static>>,
|
||||
partial: &Option<(u32, NonZeroU32)>,
|
||||
peek: &bool,
|
||||
) -> Result<(MessageDataItem<'static>, SeenFlag)> {
|
||||
// Manage Seen flag
|
||||
let mut seen = SeenFlag::DoNothing;
|
||||
if !peek && self.is_not_yet_seen() {
|
||||
// Add \Seen flag
|
||||
//self.mailbox.add_flags(uuid, &[seen_flag]).await?;
|
||||
seen = SeenFlag::MustAdd;
|
||||
}
|
||||
|
||||
// Process message
|
||||
let (text, origin) =
|
||||
match mime_view::body_ext(self.content.as_anypart()?, section, partial)? {
|
||||
mime_view::BodySection::Full(body) => (body, None),
|
||||
mime_view::BodySection::Slice { body, origin_octet } => (body, Some(origin_octet)),
|
||||
};
|
||||
|
||||
let data: NString = text.to_vec().try_into()?;
|
||||
|
||||
return Ok((
|
||||
MessageDataItem::BodyExt {
|
||||
section: section.as_ref().map(|fs| fs.clone()),
|
||||
origin,
|
||||
data,
|
||||
},
|
||||
seen,
|
||||
));
|
||||
}
|
||||
|
||||
fn internal_date(&self) -> Result<MessageDataItem<'static>> {
|
||||
let dt = Utc
|
||||
.fix()
|
||||
.timestamp_opt(
|
||||
i64::try_from(
|
||||
self.query_result
|
||||
.metadata()
|
||||
.ok_or(anyhow!("mail metadata were not fetched"))?
|
||||
.internaldate
|
||||
/ 1000,
|
||||
)?,
|
||||
0,
|
||||
)
|
||||
.earliest()
|
||||
.ok_or(anyhow!("Unable to parse internal date"))?;
|
||||
Ok(MessageDataItem::InternalDate(DateTime::unvalidated(dt)))
|
||||
}
|
||||
|
||||
fn modseq(&self) -> MessageDataItem<'static> {
|
||||
MessageDataItem::ModSeq(self.in_idx.modseq)
|
||||
}
|
||||
}
|
||||
|
||||
pub enum SeenFlag {
|
||||
DoNothing,
|
||||
MustAdd,
|
||||
}
|
||||
|
||||
// -------------------
|
||||
|
||||
pub enum FetchedMail<'a> {
|
||||
IndexOnly,
|
||||
Partial(AnyPart<'a>),
|
||||
Full(AnyPart<'a>),
|
||||
}
|
||||
impl<'a> FetchedMail<'a> {
|
||||
pub fn full_from_message(msg: Message<'a>) -> Self {
|
||||
Self::Full(AnyPart::Msg(msg))
|
||||
}
|
||||
|
||||
pub fn partial_from_message(msg: Message<'a>) -> Self {
|
||||
Self::Partial(AnyPart::Msg(msg))
|
||||
}
|
||||
|
||||
pub fn as_anypart(&self) -> Result<&AnyPart<'a>> {
|
||||
match self {
|
||||
FetchedMail::Full(x) => Ok(&x),
|
||||
FetchedMail::Partial(x) => Ok(&x),
|
||||
_ => bail!("The full message must be fetched, not only its headers"),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn as_msg(&self) -> Result<&Message<'a>> {
|
||||
match self {
|
||||
FetchedMail::Full(AnyPart::Msg(x)) => Ok(&x),
|
||||
FetchedMail::Partial(AnyPart::Msg(x)) => Ok(&x),
|
||||
_ => bail!("The full message must be fetched, not only its headers AND it must be an AnyPart::Msg."),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn as_imf(&self) -> Option<&imf::Imf<'a>> {
|
||||
match self {
|
||||
FetchedMail::Full(AnyPart::Msg(x)) => Some(&x.imf),
|
||||
FetchedMail::Partial(AnyPart::Msg(x)) => Some(&x.imf),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,772 +0,0 @@
|
|||
use std::collections::HashSet;
|
||||
use std::num::{NonZeroU32, NonZeroU64};
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::{anyhow, Error, Result};
|
||||
|
||||
use futures::stream::{StreamExt, TryStreamExt};
|
||||
|
||||
use imap_codec::imap_types::core::{Charset, Vec1};
|
||||
use imap_codec::imap_types::fetch::MessageDataItem;
|
||||
use imap_codec::imap_types::flag::{Flag, FlagFetch, FlagPerm, StoreResponse, StoreType};
|
||||
use imap_codec::imap_types::response::{Code, CodeOther, Data, Status};
|
||||
use imap_codec::imap_types::search::SearchKey;
|
||||
use imap_codec::imap_types::sequence::SequenceSet;
|
||||
|
||||
use crate::mail::mailbox::Mailbox;
|
||||
use crate::mail::query::QueryScope;
|
||||
use crate::mail::snapshot::FrozenMailbox;
|
||||
use crate::mail::uidindex::{ImapUid, ImapUidvalidity, ModSeq};
|
||||
use crate::mail::unique_ident::UniqueIdent;
|
||||
|
||||
use crate::imap::attributes::AttributesProxy;
|
||||
use crate::imap::flags;
|
||||
use crate::imap::index::Index;
|
||||
use crate::imap::mail_view::{MailView, SeenFlag};
|
||||
use crate::imap::response::Body;
|
||||
use crate::imap::search;
|
||||
|
||||
const DEFAULT_FLAGS: [Flag; 5] = [
|
||||
Flag::Seen,
|
||||
Flag::Answered,
|
||||
Flag::Flagged,
|
||||
Flag::Deleted,
|
||||
Flag::Draft,
|
||||
];
|
||||
|
||||
pub struct UpdateParameters {
|
||||
pub silence: HashSet<UniqueIdent>,
|
||||
pub with_modseq: bool,
|
||||
pub with_uid: bool,
|
||||
}
|
||||
impl Default for UpdateParameters {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
silence: HashSet::new(),
|
||||
with_modseq: false,
|
||||
with_uid: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A MailboxView is responsible for giving the client the information
|
||||
/// it needs about a mailbox, such as an initial summary of the mailbox's
|
||||
/// content and continuous updates indicating when the content
|
||||
/// of the mailbox has been changed.
|
||||
/// To do this, it keeps a variable `known_state` that corresponds to
|
||||
/// what the client knows, and produces IMAP messages to be sent to the
|
||||
/// client that go along updates to `known_state`.
|
||||
pub struct MailboxView {
|
||||
pub internal: FrozenMailbox,
|
||||
pub is_condstore: bool,
|
||||
}
|
||||
|
||||
impl MailboxView {
|
||||
/// Creates a new IMAP view into a mailbox.
|
||||
pub async fn new(mailbox: Arc<Mailbox>, is_cond: bool) -> Self {
|
||||
Self {
|
||||
internal: mailbox.frozen().await,
|
||||
is_condstore: is_cond,
|
||||
}
|
||||
}
|
||||
|
||||
/// Create an updated view, useful to make a diff
|
||||
/// between what the client knows and new stuff
|
||||
/// Produces a set of IMAP responses describing the change between
|
||||
/// what the client knows and what is actually in the mailbox.
|
||||
/// This does NOT trigger a sync, it bases itself on what is currently
|
||||
/// loaded in RAM by Bayou.
|
||||
pub async fn update(&mut self, params: UpdateParameters) -> Result<Vec<Body<'static>>> {
|
||||
let old_snapshot = self.internal.update().await;
|
||||
let new_snapshot = &self.internal.snapshot;
|
||||
|
||||
let mut data = Vec::<Body>::new();
|
||||
|
||||
// Calculate diff between two mailbox states
|
||||
// See example in IMAP RFC in section on NOOP command:
|
||||
// we want to produce something like this:
|
||||
// C: a047 NOOP
|
||||
// S: * 22 EXPUNGE
|
||||
// S: * 23 EXISTS
|
||||
// S: * 14 FETCH (UID 1305 FLAGS (\Seen \Deleted))
|
||||
// S: a047 OK Noop completed
|
||||
// In other words:
|
||||
// - notify client of expunged mails
|
||||
// - if new mails arrived, notify client of number of existing mails
|
||||
// - if flags changed for existing mails, tell client
|
||||
// (for this last step: if uidvalidity changed, do nothing,
|
||||
// just notify of new uidvalidity and they will resync)
|
||||
|
||||
// - notify client of expunged mails
|
||||
let mut n_expunge = 0;
|
||||
for (i, (_uid, uuid)) in old_snapshot.idx_by_uid.iter().enumerate() {
|
||||
if !new_snapshot.table.contains_key(uuid) {
|
||||
data.push(Body::Data(Data::Expunge(
|
||||
NonZeroU32::try_from((i + 1 - n_expunge) as u32).unwrap(),
|
||||
)));
|
||||
n_expunge += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// - if new mails arrived, notify client of number of existing mails
|
||||
if new_snapshot.table.len() != old_snapshot.table.len() - n_expunge
|
||||
|| new_snapshot.uidvalidity != old_snapshot.uidvalidity
|
||||
{
|
||||
data.push(self.exists_status()?);
|
||||
}
|
||||
|
||||
if new_snapshot.uidvalidity != old_snapshot.uidvalidity {
|
||||
// TODO: do we want to push less/more info than this?
|
||||
data.push(self.uidvalidity_status()?);
|
||||
data.push(self.uidnext_status()?);
|
||||
} else {
|
||||
// - if flags changed for existing mails, tell client
|
||||
for (i, (_uid, uuid)) in new_snapshot.idx_by_uid.iter().enumerate() {
|
||||
if params.silence.contains(uuid) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let old_mail = old_snapshot.table.get(uuid);
|
||||
let new_mail = new_snapshot.table.get(uuid);
|
||||
if old_mail.is_some() && old_mail != new_mail {
|
||||
if let Some((uid, modseq, flags)) = new_mail {
|
||||
let mut items = vec![MessageDataItem::Flags(
|
||||
flags.iter().filter_map(|f| flags::from_str(f)).collect(),
|
||||
)];
|
||||
|
||||
if params.with_uid {
|
||||
items.push(MessageDataItem::Uid(*uid));
|
||||
}
|
||||
|
||||
if params.with_modseq {
|
||||
items.push(MessageDataItem::ModSeq(*modseq));
|
||||
}
|
||||
|
||||
data.push(Body::Data(Data::Fetch {
|
||||
seq: NonZeroU32::try_from((i + 1) as u32).unwrap(),
|
||||
items: items.try_into()?,
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
/// Generates the necessary IMAP messages so that the client
|
||||
/// has a satisfactory summary of the current mailbox's state.
|
||||
/// These are the messages that are sent in response to a SELECT command.
|
||||
pub fn summary(&self) -> Result<Vec<Body<'static>>> {
|
||||
let mut data = Vec::<Body>::new();
|
||||
data.push(self.exists_status()?);
|
||||
data.push(self.recent_status()?);
|
||||
data.extend(self.flags_status()?.into_iter());
|
||||
data.push(self.uidvalidity_status()?);
|
||||
data.push(self.uidnext_status()?);
|
||||
if self.is_condstore {
|
||||
data.push(self.highestmodseq_status()?);
|
||||
}
|
||||
/*self.unseen_first_status()?
|
||||
.map(|unseen_status| data.push(unseen_status));*/
|
||||
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
pub async fn store<'a>(
|
||||
&mut self,
|
||||
sequence_set: &SequenceSet,
|
||||
kind: &StoreType,
|
||||
response: &StoreResponse,
|
||||
flags: &[Flag<'a>],
|
||||
unchanged_since: Option<NonZeroU64>,
|
||||
is_uid_store: &bool,
|
||||
) -> Result<(Vec<Body<'static>>, Vec<NonZeroU32>)> {
|
||||
self.internal.sync().await?;
|
||||
|
||||
let flags = flags.iter().map(|x| x.to_string()).collect::<Vec<_>>();
|
||||
|
||||
let idx = self.index()?;
|
||||
let (editable, in_conflict) =
|
||||
idx.fetch_unchanged_since(sequence_set, unchanged_since, *is_uid_store)?;
|
||||
|
||||
for mi in editable.iter() {
|
||||
match kind {
|
||||
StoreType::Add => {
|
||||
self.internal.mailbox.add_flags(mi.uuid, &flags[..]).await?;
|
||||
}
|
||||
StoreType::Remove => {
|
||||
self.internal.mailbox.del_flags(mi.uuid, &flags[..]).await?;
|
||||
}
|
||||
StoreType::Replace => {
|
||||
self.internal.mailbox.set_flags(mi.uuid, &flags[..]).await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let silence = match response {
|
||||
StoreResponse::Answer => HashSet::new(),
|
||||
StoreResponse::Silent => editable.iter().map(|midx| midx.uuid).collect(),
|
||||
};
|
||||
|
||||
let conflict_id_or_uid = match is_uid_store {
|
||||
true => in_conflict.into_iter().map(|midx| midx.uid).collect(),
|
||||
_ => in_conflict.into_iter().map(|midx| midx.i).collect(),
|
||||
};
|
||||
|
||||
let summary = self
|
||||
.update(UpdateParameters {
|
||||
with_uid: *is_uid_store,
|
||||
with_modseq: unchanged_since.is_some(),
|
||||
silence,
|
||||
})
|
||||
.await?;
|
||||
|
||||
Ok((summary, conflict_id_or_uid))
|
||||
}
|
||||
|
||||
pub async fn idle_sync(&mut self) -> Result<Vec<Body<'static>>> {
|
||||
self.internal
|
||||
.mailbox
|
||||
.notify()
|
||||
.await
|
||||
.upgrade()
|
||||
.ok_or(anyhow!("test"))?
|
||||
.notified()
|
||||
.await;
|
||||
self.internal.mailbox.opportunistic_sync().await?;
|
||||
self.update(UpdateParameters::default()).await
|
||||
}
|
||||
|
||||
pub async fn expunge(
|
||||
&mut self,
|
||||
maybe_seq_set: &Option<SequenceSet>,
|
||||
) -> Result<Vec<Body<'static>>> {
|
||||
// Get a recent view to apply our change
|
||||
self.internal.sync().await?;
|
||||
let state = self.internal.peek().await;
|
||||
let idx = Index::new(&state)?;
|
||||
|
||||
// Build a default sequence set for the default case
|
||||
use imap_codec::imap_types::sequence::{SeqOrUid, Sequence};
|
||||
let seq = match maybe_seq_set {
|
||||
Some(s) => s.clone(),
|
||||
None => SequenceSet(
|
||||
vec![Sequence::Range(
|
||||
SeqOrUid::Value(NonZeroU32::MIN),
|
||||
SeqOrUid::Asterisk,
|
||||
)]
|
||||
.try_into()
|
||||
.unwrap(),
|
||||
),
|
||||
};
|
||||
|
||||
let deleted_flag = Flag::Deleted.to_string();
|
||||
let msgs = idx
|
||||
.fetch_on_uid(&seq)
|
||||
.into_iter()
|
||||
.filter(|midx| midx.flags.iter().any(|x| *x == deleted_flag))
|
||||
.map(|midx| midx.uuid);
|
||||
|
||||
for msg in msgs {
|
||||
self.internal.mailbox.delete(msg).await?;
|
||||
}
|
||||
|
||||
self.update(UpdateParameters::default()).await
|
||||
}
|
||||
|
||||
pub async fn copy(
|
||||
&self,
|
||||
sequence_set: &SequenceSet,
|
||||
to: Arc<Mailbox>,
|
||||
is_uid_copy: &bool,
|
||||
) -> Result<(ImapUidvalidity, Vec<(ImapUid, ImapUid)>)> {
|
||||
let idx = self.index()?;
|
||||
let mails = idx.fetch(sequence_set, *is_uid_copy)?;
|
||||
|
||||
let mut new_uuids = vec![];
|
||||
for mi in mails.iter() {
|
||||
new_uuids.push(to.copy_from(&self.internal.mailbox, mi.uuid).await?);
|
||||
}
|
||||
|
||||
let mut ret = vec![];
|
||||
let to_state = to.current_uid_index().await;
|
||||
for (mi, new_uuid) in mails.iter().zip(new_uuids.iter()) {
|
||||
let dest_uid = to_state
|
||||
.table
|
||||
.get(new_uuid)
|
||||
.ok_or(anyhow!("copied mail not in destination mailbox"))?
|
||||
.0;
|
||||
ret.push((mi.uid, dest_uid));
|
||||
}
|
||||
|
||||
Ok((to_state.uidvalidity, ret))
|
||||
}
|
||||
|
||||
pub async fn r#move(
|
||||
&mut self,
|
||||
sequence_set: &SequenceSet,
|
||||
to: Arc<Mailbox>,
|
||||
is_uid_copy: &bool,
|
||||
) -> Result<(ImapUidvalidity, Vec<(ImapUid, ImapUid)>, Vec<Body<'static>>)> {
|
||||
let idx = self.index()?;
|
||||
let mails = idx.fetch(sequence_set, *is_uid_copy)?;
|
||||
|
||||
for mi in mails.iter() {
|
||||
to.move_from(&self.internal.mailbox, mi.uuid).await?;
|
||||
}
|
||||
|
||||
let mut ret = vec![];
|
||||
let to_state = to.current_uid_index().await;
|
||||
for mi in mails.iter() {
|
||||
let dest_uid = to_state
|
||||
.table
|
||||
.get(&mi.uuid)
|
||||
.ok_or(anyhow!("moved mail not in destination mailbox"))?
|
||||
.0;
|
||||
ret.push((mi.uid, dest_uid));
|
||||
}
|
||||
|
||||
let update = self
|
||||
.update(UpdateParameters {
|
||||
with_uid: *is_uid_copy,
|
||||
..UpdateParameters::default()
|
||||
})
|
||||
.await?;
|
||||
|
||||
Ok((to_state.uidvalidity, ret, update))
|
||||
}
|
||||
|
||||
/// Looks up state changes in the mailbox and produces a set of IMAP
|
||||
/// responses describing the new state.
|
||||
pub async fn fetch<'b>(
|
||||
&self,
|
||||
sequence_set: &SequenceSet,
|
||||
ap: &AttributesProxy,
|
||||
changed_since: Option<NonZeroU64>,
|
||||
is_uid_fetch: &bool,
|
||||
) -> Result<Vec<Body<'static>>> {
|
||||
// [1/6] Pre-compute data
|
||||
// a. what are the uuids of the emails we want?
|
||||
// b. do we need to fetch the full body?
|
||||
//let ap = AttributesProxy::new(attributes, *is_uid_fetch);
|
||||
let query_scope = match ap.need_body() {
|
||||
true => QueryScope::Full,
|
||||
_ => QueryScope::Partial,
|
||||
};
|
||||
tracing::debug!("Query scope {:?}", query_scope);
|
||||
let idx = self.index()?;
|
||||
let mail_idx_list = idx.fetch_changed_since(sequence_set, changed_since, *is_uid_fetch)?;
|
||||
|
||||
// [2/6] Fetch the emails
|
||||
let uuids = mail_idx_list
|
||||
.iter()
|
||||
.map(|midx| midx.uuid)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let query = self.internal.query(&uuids, query_scope);
|
||||
//let query_result = self.internal.query(&uuids, query_scope).fetch().await?;
|
||||
|
||||
let query_stream = query
|
||||
.fetch()
|
||||
.zip(futures::stream::iter(mail_idx_list))
|
||||
// [3/6] Derive an IMAP-specific view from the results, apply the filters
|
||||
.map(|(maybe_qr, midx)| match maybe_qr {
|
||||
Ok(qr) => Ok((MailView::new(&qr, midx)?.filter(&ap)?, midx)),
|
||||
Err(e) => Err(e),
|
||||
})
|
||||
// [4/6] Apply the IMAP transformation
|
||||
.then(|maybe_ret| async move {
|
||||
let ((body, seen), midx) = maybe_ret?;
|
||||
|
||||
// [5/6] Register the \Seen flags
|
||||
if matches!(seen, SeenFlag::MustAdd) {
|
||||
let seen_flag = Flag::Seen.to_string();
|
||||
self.internal
|
||||
.mailbox
|
||||
.add_flags(midx.uuid, &[seen_flag])
|
||||
.await?;
|
||||
}
|
||||
|
||||
Ok::<_, anyhow::Error>(body)
|
||||
});
|
||||
|
||||
// [6/6] Build the final result that will be sent to the client.
|
||||
query_stream.try_collect().await
|
||||
}
|
||||
|
||||
/// A naive search implementation...
|
||||
pub async fn search<'a>(
|
||||
&self,
|
||||
_charset: &Option<Charset<'a>>,
|
||||
search_key: &SearchKey<'a>,
|
||||
uid: bool,
|
||||
) -> Result<(Vec<Body<'static>>, bool)> {
|
||||
// 1. Compute the subset of sequence identifiers we need to fetch
|
||||
// based on the search query
|
||||
let crit = search::Criteria(search_key);
|
||||
let (seq_set, seq_type) = crit.to_sequence_set();
|
||||
|
||||
// 2. Get the selection
|
||||
let idx = self.index()?;
|
||||
let selection = idx.fetch(&seq_set, seq_type.is_uid())?;
|
||||
|
||||
// 3. Filter the selection based on the ID / UID / Flags
|
||||
let (kept_idx, to_fetch) = crit.filter_on_idx(&selection);
|
||||
|
||||
// 4.a Fetch additional info about the emails
|
||||
let query_scope = crit.query_scope();
|
||||
let uuids = to_fetch.iter().map(|midx| midx.uuid).collect::<Vec<_>>();
|
||||
let query = self.internal.query(&uuids, query_scope);
|
||||
|
||||
// 4.b We don't want to keep all data in memory, so we do the computing in a stream
|
||||
let query_stream = query
|
||||
.fetch()
|
||||
.zip(futures::stream::iter(&to_fetch))
|
||||
// 5.a Build a mailview with the body, might fail with an error
|
||||
// 5.b If needed, filter the selection based on the body, but keep the errors
|
||||
// 6. Drop the query+mailbox, keep only the mail index
|
||||
// Here we release a lot of memory, this is the most important part ^^
|
||||
.filter_map(|(maybe_qr, midx)| {
|
||||
let r = match maybe_qr {
|
||||
Ok(qr) => match MailView::new(&qr, midx).map(|mv| crit.is_keep_on_query(&mv)) {
|
||||
Ok(true) => Some(Ok(*midx)),
|
||||
Ok(_) => None,
|
||||
Err(e) => Some(Err(e)),
|
||||
},
|
||||
Err(e) => Some(Err(e)),
|
||||
};
|
||||
futures::future::ready(r)
|
||||
});
|
||||
|
||||
// 7. Chain both streams (part resolved from index, part resolved from metadata+body)
|
||||
let main_stream = futures::stream::iter(kept_idx)
|
||||
.map(Ok)
|
||||
.chain(query_stream)
|
||||
.map_ok(|idx| match uid {
|
||||
true => (idx.uid, idx.modseq),
|
||||
_ => (idx.i, idx.modseq),
|
||||
});
|
||||
|
||||
// 8. Do the actual computation
|
||||
let internal_result: Vec<_> = main_stream.try_collect().await?;
|
||||
let (selection, modseqs): (Vec<_>, Vec<_>) = internal_result.into_iter().unzip();
|
||||
|
||||
// 9. Aggregate the maximum modseq value
|
||||
let maybe_modseq = match crit.is_modseq() {
|
||||
true => modseqs.into_iter().max(),
|
||||
_ => None,
|
||||
};
|
||||
|
||||
// 10. Return the final result
|
||||
Ok((
|
||||
vec![Body::Data(Data::Search(selection, maybe_modseq))],
|
||||
maybe_modseq.is_some(),
|
||||
))
|
||||
}
|
||||
|
||||
// ----
|
||||
/// @FIXME index should be stored for longer than a single request
|
||||
/// Instead they should be tied to the FrozenMailbox refresh
|
||||
/// It's not trivial to refactor the code to do that, so we are doing
|
||||
/// some useless computation for now...
|
||||
fn index<'a>(&'a self) -> Result<Index<'a>> {
|
||||
Index::new(&self.internal.snapshot)
|
||||
}
|
||||
|
||||
/// Produce an OK [UIDVALIDITY _] message corresponding to `known_state`
|
||||
fn uidvalidity_status(&self) -> Result<Body<'static>> {
|
||||
let uid_validity = Status::ok(
|
||||
None,
|
||||
Some(Code::UidValidity(self.uidvalidity())),
|
||||
"UIDs valid",
|
||||
)
|
||||
.map_err(Error::msg)?;
|
||||
Ok(Body::Status(uid_validity))
|
||||
}
|
||||
|
||||
pub(crate) fn uidvalidity(&self) -> ImapUidvalidity {
|
||||
self.internal.snapshot.uidvalidity
|
||||
}
|
||||
|
||||
/// Produce an OK [UIDNEXT _] message corresponding to `known_state`
|
||||
fn uidnext_status(&self) -> Result<Body<'static>> {
|
||||
let next_uid = Status::ok(
|
||||
None,
|
||||
Some(Code::UidNext(self.uidnext())),
|
||||
"Predict next UID",
|
||||
)
|
||||
.map_err(Error::msg)?;
|
||||
Ok(Body::Status(next_uid))
|
||||
}
|
||||
|
||||
pub(crate) fn uidnext(&self) -> ImapUid {
|
||||
self.internal.snapshot.uidnext
|
||||
}
|
||||
|
||||
pub(crate) fn highestmodseq_status(&self) -> Result<Body<'static>> {
|
||||
Ok(Body::Status(Status::ok(
|
||||
None,
|
||||
Some(Code::Other(CodeOther::unvalidated(
|
||||
format!("HIGHESTMODSEQ {}", self.highestmodseq()).into_bytes(),
|
||||
))),
|
||||
"Highest",
|
||||
)?))
|
||||
}
|
||||
|
||||
pub(crate) fn highestmodseq(&self) -> ModSeq {
|
||||
self.internal.snapshot.highestmodseq
|
||||
}
|
||||
|
||||
/// Produce an EXISTS message corresponding to the number of mails
|
||||
/// in `known_state`
|
||||
fn exists_status(&self) -> Result<Body<'static>> {
|
||||
Ok(Body::Data(Data::Exists(self.exists()?)))
|
||||
}
|
||||
|
||||
pub(crate) fn exists(&self) -> Result<u32> {
|
||||
Ok(u32::try_from(self.internal.snapshot.idx_by_uid.len())?)
|
||||
}
|
||||
|
||||
/// Produce a RECENT message corresponding to the number of
|
||||
/// recent mails in `known_state`
|
||||
fn recent_status(&self) -> Result<Body<'static>> {
|
||||
Ok(Body::Data(Data::Recent(self.recent()?)))
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
fn unseen_first_status(&self) -> Result<Option<Body<'static>>> {
|
||||
Ok(self
|
||||
.unseen_first()?
|
||||
.map(|unseen_id| {
|
||||
Status::ok(None, Some(Code::Unseen(unseen_id)), "First unseen.").map(Body::Status)
|
||||
})
|
||||
.transpose()?)
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
fn unseen_first(&self) -> Result<Option<NonZeroU32>> {
|
||||
Ok(self
|
||||
.internal
|
||||
.snapshot
|
||||
.table
|
||||
.values()
|
||||
.enumerate()
|
||||
.find(|(_i, (_imap_uid, _modseq, flags))| !flags.contains(&"\\Seen".to_string()))
|
||||
.map(|(i, _)| NonZeroU32::try_from(i as u32 + 1))
|
||||
.transpose()?)
|
||||
}
|
||||
|
||||
pub(crate) fn recent(&self) -> Result<u32> {
|
||||
let recent = self
|
||||
.internal
|
||||
.snapshot
|
||||
.idx_by_flag
|
||||
.get(&"\\Recent".to_string())
|
||||
.map(|os| os.len())
|
||||
.unwrap_or(0);
|
||||
Ok(u32::try_from(recent)?)
|
||||
}
|
||||
|
||||
/// Produce a FLAGS and a PERMANENTFLAGS message that indicates
|
||||
/// the flags that are in `known_state` + default flags
|
||||
fn flags_status(&self) -> Result<Vec<Body<'static>>> {
|
||||
let mut body = vec![];
|
||||
|
||||
// 1. Collecting all the possible flags in the mailbox
|
||||
// 1.a Fetch them from our index
|
||||
let mut known_flags: Vec<Flag> = self
|
||||
.internal
|
||||
.snapshot
|
||||
.idx_by_flag
|
||||
.flags()
|
||||
.filter_map(|f| match flags::from_str(f) {
|
||||
Some(FlagFetch::Flag(fl)) => Some(fl),
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
// 1.b Merge it with our default flags list
|
||||
for f in DEFAULT_FLAGS.iter() {
|
||||
if !known_flags.contains(f) {
|
||||
known_flags.push(f.clone());
|
||||
}
|
||||
}
|
||||
// 1.c Create the IMAP message
|
||||
body.push(Body::Data(Data::Flags(known_flags.clone())));
|
||||
|
||||
// 2. Returning flags that are persisted
|
||||
// 2.a Always advertise our default flags
|
||||
let mut permanent = DEFAULT_FLAGS
|
||||
.iter()
|
||||
.map(|f| FlagPerm::Flag(f.clone()))
|
||||
.collect::<Vec<_>>();
|
||||
// 2.b Say that we support any keyword flag
|
||||
permanent.push(FlagPerm::Asterisk);
|
||||
// 2.c Create the IMAP message
|
||||
let permanent_flags = Status::ok(
|
||||
None,
|
||||
Some(Code::PermanentFlags(permanent)),
|
||||
"Flags permitted",
|
||||
)
|
||||
.map_err(Error::msg)?;
|
||||
body.push(Body::Status(permanent_flags));
|
||||
|
||||
// Done!
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
pub(crate) fn unseen_count(&self) -> usize {
|
||||
let total = self.internal.snapshot.table.len();
|
||||
let seen = self
|
||||
.internal
|
||||
.snapshot
|
||||
.idx_by_flag
|
||||
.get(&Flag::Seen.to_string())
|
||||
.map(|x| x.len())
|
||||
.unwrap_or(0);
|
||||
total - seen
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use imap_codec::encode::Encoder;
|
||||
use imap_codec::imap_types::core::Vec1;
|
||||
use imap_codec::imap_types::fetch::Section;
|
||||
use imap_codec::imap_types::fetch::{MacroOrMessageDataItemNames, MessageDataItemName};
|
||||
use imap_codec::imap_types::response::Response;
|
||||
use imap_codec::ResponseCodec;
|
||||
use std::fs;
|
||||
|
||||
use crate::cryptoblob;
|
||||
use crate::imap::index::MailIndex;
|
||||
use crate::imap::mail_view::MailView;
|
||||
use crate::imap::mime_view;
|
||||
use crate::mail::mailbox::MailMeta;
|
||||
use crate::mail::query::QueryResult;
|
||||
use crate::mail::unique_ident;
|
||||
|
||||
#[test]
|
||||
fn mailview_body_ext() -> Result<()> {
|
||||
let ap = AttributesProxy::new(
|
||||
&MacroOrMessageDataItemNames::MessageDataItemNames(vec![
|
||||
MessageDataItemName::BodyExt {
|
||||
section: Some(Section::Header(None)),
|
||||
partial: None,
|
||||
peek: false,
|
||||
},
|
||||
]),
|
||||
&[],
|
||||
false,
|
||||
);
|
||||
|
||||
let key = cryptoblob::gen_key();
|
||||
let meta = MailMeta {
|
||||
internaldate: 0u64,
|
||||
headers: vec![],
|
||||
message_key: key,
|
||||
rfc822_size: 8usize,
|
||||
};
|
||||
|
||||
let index_entry = (NonZeroU32::MIN, NonZeroU64::MIN, vec![]);
|
||||
let mail_in_idx = MailIndex {
|
||||
i: NonZeroU32::MIN,
|
||||
uid: index_entry.0,
|
||||
modseq: index_entry.1,
|
||||
uuid: unique_ident::gen_ident(),
|
||||
flags: &index_entry.2,
|
||||
};
|
||||
let rfc822 = b"Subject: hello\r\nFrom: a@a.a\r\nTo: b@b.b\r\nDate: Thu, 12 Oct 2023 08:45:28 +0000\r\n\r\nhello world";
|
||||
let qr = QueryResult::FullResult {
|
||||
uuid: mail_in_idx.uuid.clone(),
|
||||
metadata: meta,
|
||||
content: rfc822.to_vec(),
|
||||
};
|
||||
|
||||
let mv = MailView::new(&qr, &mail_in_idx)?;
|
||||
let (res_body, _seen) = mv.filter(&ap)?;
|
||||
|
||||
let fattr = match res_body {
|
||||
Body::Data(Data::Fetch {
|
||||
seq: _seq,
|
||||
items: attr,
|
||||
}) => Ok(attr),
|
||||
_ => Err(anyhow!("Not a fetch body")),
|
||||
}?;
|
||||
|
||||
assert_eq!(fattr.as_ref().len(), 1);
|
||||
|
||||
let (sec, _orig, _data) = match &fattr.as_ref()[0] {
|
||||
MessageDataItem::BodyExt {
|
||||
section,
|
||||
origin,
|
||||
data,
|
||||
} => Ok((section, origin, data)),
|
||||
_ => Err(anyhow!("not a body ext message attribute")),
|
||||
}?;
|
||||
|
||||
assert_eq!(sec.as_ref().unwrap(), &Section::Header(None));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Future automated test. We use lossy utf8 conversion + lowercase everything,
|
||||
/// so this test might allow invalid results. But at least it allows us to quickly test a
|
||||
/// large variety of emails.
|
||||
/// Keep in mind that special cases must still be tested manually!
|
||||
#[test]
|
||||
fn fetch_body() -> Result<()> {
|
||||
let prefixes = [
|
||||
/* *** MY OWN DATASET *** */
|
||||
"tests/emails/dxflrs/0001_simple",
|
||||
"tests/emails/dxflrs/0002_mime",
|
||||
"tests/emails/dxflrs/0003_mime-in-mime",
|
||||
"tests/emails/dxflrs/0004_msg-in-msg",
|
||||
// eml_codec do not support continuation for the moment
|
||||
//"tests/emails/dxflrs/0005_mail-parser-readme",
|
||||
"tests/emails/dxflrs/0006_single-mime",
|
||||
"tests/emails/dxflrs/0007_raw_msg_in_rfc822",
|
||||
/* *** (STRANGE) RFC *** */
|
||||
//"tests/emails/rfc/000", // must return text/enriched, we return text/plain
|
||||
//"tests/emails/rfc/001", // does not recognize the multipart/external-body, breaks the
|
||||
// whole parsing
|
||||
//"tests/emails/rfc/002", // wrong date in email
|
||||
|
||||
//"tests/emails/rfc/003", // dovecot fixes \r\r: the bytes number is wrong + text/enriched
|
||||
|
||||
/* *** THIRD PARTY *** */
|
||||
//"tests/emails/thirdparty/000", // dovecot fixes \r\r: the bytes number is wrong
|
||||
//"tests/emails/thirdparty/001", // same
|
||||
"tests/emails/thirdparty/002", // same
|
||||
|
||||
/* *** LEGACY *** */
|
||||
//"tests/emails/legacy/000", // same issue with \r\r
|
||||
];
|
||||
|
||||
for pref in prefixes.iter() {
|
||||
println!("{}", pref);
|
||||
let txt = fs::read(format!("{}.eml", pref))?;
|
||||
let oracle = fs::read(format!("{}.dovecot.body", pref))?;
|
||||
let message = eml_codec::parse_message(&txt).unwrap().1;
|
||||
|
||||
let test_repr = Response::Data(Data::Fetch {
|
||||
seq: NonZeroU32::new(1).unwrap(),
|
||||
items: Vec1::from(MessageDataItem::Body(mime_view::bodystructure(
|
||||
&message.child,
|
||||
false,
|
||||
)?)),
|
||||
});
|
||||
let test_bytes = ResponseCodec::new().encode(&test_repr).dump();
|
||||
let test_str = String::from_utf8_lossy(&test_bytes).to_lowercase();
|
||||
|
||||
let oracle_str =
|
||||
format!("* 1 FETCH {}\r\n", String::from_utf8_lossy(&oracle)).to_lowercase();
|
||||
|
||||
println!("aerogramme: {}\n\ndovecot: {}\n\n", test_str, oracle_str);
|
||||
//println!("\n\n {} \n\n", String::from_utf8_lossy(&resp));
|
||||
assert_eq!(test_str, oracle_str);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
@ -1,580 +0,0 @@
|
|||
use std::borrow::Cow;
|
||||
use std::collections::HashSet;
|
||||
use std::num::NonZeroU32;
|
||||
|
||||
use anyhow::{anyhow, bail, Result};
|
||||
|
||||
use imap_codec::imap_types::body::{
|
||||
BasicFields, Body as FetchBody, BodyStructure, MultiPartExtensionData, SinglePartExtensionData,
|
||||
SpecificFields,
|
||||
};
|
||||
use imap_codec::imap_types::core::{AString, IString, NString, Vec1};
|
||||
use imap_codec::imap_types::fetch::{Part as FetchPart, Section as FetchSection};
|
||||
|
||||
use eml_codec::{
|
||||
header, mime, mime::r#type::Deductible, part::composite, part::discrete, part::AnyPart,
|
||||
};
|
||||
|
||||
use crate::imap::imf_view::ImfView;
|
||||
|
||||
pub enum BodySection<'a> {
|
||||
Full(Cow<'a, [u8]>),
|
||||
Slice {
|
||||
body: Cow<'a, [u8]>,
|
||||
origin_octet: u32,
|
||||
},
|
||||
}
|
||||
|
||||
/// Logic for BODY[<section>]<<partial>>
|
||||
/// Works in 3 times:
|
||||
/// 1. Find the section (RootMime::subset)
|
||||
/// 2. Apply the extraction logic (SelectedMime::extract), like TEXT, HEADERS, etc.
|
||||
/// 3. Keep only the given subset provided by partial
|
||||
///
|
||||
/// Example of message sections:
|
||||
///
|
||||
/// ```
|
||||
/// HEADER ([RFC-2822] header of the message)
|
||||
/// TEXT ([RFC-2822] text body of the message) MULTIPART/MIXED
|
||||
/// 1 TEXT/PLAIN
|
||||
/// 2 APPLICATION/OCTET-STREAM
|
||||
/// 3 MESSAGE/RFC822
|
||||
/// 3.HEADER ([RFC-2822] header of the message)
|
||||
/// 3.TEXT ([RFC-2822] text body of the message) MULTIPART/MIXED
|
||||
/// 3.1 TEXT/PLAIN
|
||||
/// 3.2 APPLICATION/OCTET-STREAM
|
||||
/// 4 MULTIPART/MIXED
|
||||
/// 4.1 IMAGE/GIF
|
||||
/// 4.1.MIME ([MIME-IMB] header for the IMAGE/GIF)
|
||||
/// 4.2 MESSAGE/RFC822
|
||||
/// 4.2.HEADER ([RFC-2822] header of the message)
|
||||
/// 4.2.TEXT ([RFC-2822] text body of the message) MULTIPART/MIXED
|
||||
/// 4.2.1 TEXT/PLAIN
|
||||
/// 4.2.2 MULTIPART/ALTERNATIVE
|
||||
/// 4.2.2.1 TEXT/PLAIN
|
||||
/// 4.2.2.2 TEXT/RICHTEXT
|
||||
/// ```
|
||||
pub fn body_ext<'a>(
|
||||
part: &'a AnyPart<'a>,
|
||||
section: &'a Option<FetchSection<'a>>,
|
||||
partial: &'a Option<(u32, NonZeroU32)>,
|
||||
) -> Result<BodySection<'a>> {
|
||||
let root_mime = NodeMime(part);
|
||||
let (extractor, path) = SubsettedSection::from(section);
|
||||
let selected_mime = root_mime.subset(path)?;
|
||||
let extracted_full = selected_mime.extract(&extractor)?;
|
||||
Ok(extracted_full.to_body_section(partial))
|
||||
}
|
||||
|
||||
/// Logic for BODY and BODYSTRUCTURE
|
||||
///
|
||||
/// ```raw
|
||||
/// b fetch 29878:29879 (BODY)
|
||||
/// * 29878 FETCH (BODY (("text" "plain" ("charset" "utf-8") NIL NIL "quoted-printable" 3264 82)("text" "html" ("charset" "utf-8") NIL NIL "quoted-printable" 31834 643) "alternative"))
|
||||
/// * 29879 FETCH (BODY ("text" "html" ("charset" "us-ascii") NIL NIL "7bit" 4107 131))
|
||||
/// ^^^^^^^^^^^^^^^^^^^^^^ ^^^ ^^^ ^^^^^^ ^^^^ ^^^
|
||||
/// | | | | | | number of lines
|
||||
/// | | | | | size
|
||||
/// | | | | content transfer encoding
|
||||
/// | | | description
|
||||
/// | | id
|
||||
/// | parameter list
|
||||
/// b OK Fetch completed (0.001 + 0.000 secs).
|
||||
/// ```
|
||||
pub fn bodystructure(part: &AnyPart, is_ext: bool) -> Result<BodyStructure<'static>> {
|
||||
NodeMime(part).structure(is_ext)
|
||||
}
|
||||
|
||||
/// NodeMime
|
||||
///
|
||||
/// Used for recursive logic on MIME.
|
||||
/// See SelectedMime for inspection.
|
||||
struct NodeMime<'a>(&'a AnyPart<'a>);
|
||||
impl<'a> NodeMime<'a> {
|
||||
/// A MIME object is a tree of elements.
|
||||
/// The path indicates which element must be picked.
|
||||
/// This function returns the picked element as the new view
|
||||
fn subset(self, path: Option<&'a FetchPart>) -> Result<SelectedMime<'a>> {
|
||||
match path {
|
||||
None => Ok(SelectedMime(self.0)),
|
||||
Some(v) => self.rec_subset(v.0.as_ref()),
|
||||
}
|
||||
}
|
||||
|
||||
fn rec_subset(self, path: &'a [NonZeroU32]) -> Result<SelectedMime> {
|
||||
if path.is_empty() {
|
||||
Ok(SelectedMime(self.0))
|
||||
} else {
|
||||
match self.0 {
|
||||
AnyPart::Mult(x) => {
|
||||
let next = Self(x.children
|
||||
.get(path[0].get() as usize - 1)
|
||||
.ok_or(anyhow!("Unable to resolve subpath {:?}, current multipart has only {} elements", path, x.children.len()))?);
|
||||
next.rec_subset(&path[1..])
|
||||
},
|
||||
AnyPart::Msg(x) => {
|
||||
let next = Self(x.child.as_ref());
|
||||
next.rec_subset(path)
|
||||
},
|
||||
_ => bail!("You tried to access a subpart on an atomic part (text or binary). Unresolved subpath {:?}", path),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn structure(&self, is_ext: bool) -> Result<BodyStructure<'static>> {
|
||||
match self.0 {
|
||||
AnyPart::Txt(x) => NodeTxt(self, x).structure(is_ext),
|
||||
AnyPart::Bin(x) => NodeBin(self, x).structure(is_ext),
|
||||
AnyPart::Mult(x) => NodeMult(self, x).structure(is_ext),
|
||||
AnyPart::Msg(x) => NodeMsg(self, x).structure(is_ext),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//----------------------------------------------------------
|
||||
|
||||
/// A FetchSection must be handled in 2 times:
|
||||
/// - First we must extract the MIME part
|
||||
/// - Then we must process it as desired
|
||||
/// The given struct mixes both work, so
|
||||
/// we separate this work here.
|
||||
enum SubsettedSection<'a> {
|
||||
Part,
|
||||
Header,
|
||||
HeaderFields(&'a Vec1<AString<'a>>),
|
||||
HeaderFieldsNot(&'a Vec1<AString<'a>>),
|
||||
Text,
|
||||
Mime,
|
||||
}
|
||||
impl<'a> SubsettedSection<'a> {
|
||||
fn from(section: &'a Option<FetchSection>) -> (Self, Option<&'a FetchPart>) {
|
||||
match section {
|
||||
Some(FetchSection::Text(maybe_part)) => (Self::Text, maybe_part.as_ref()),
|
||||
Some(FetchSection::Header(maybe_part)) => (Self::Header, maybe_part.as_ref()),
|
||||
Some(FetchSection::HeaderFields(maybe_part, fields)) => {
|
||||
(Self::HeaderFields(fields), maybe_part.as_ref())
|
||||
}
|
||||
Some(FetchSection::HeaderFieldsNot(maybe_part, fields)) => {
|
||||
(Self::HeaderFieldsNot(fields), maybe_part.as_ref())
|
||||
}
|
||||
Some(FetchSection::Mime(part)) => (Self::Mime, Some(part)),
|
||||
Some(FetchSection::Part(part)) => (Self::Part, Some(part)),
|
||||
None => (Self::Part, None),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Used for current MIME inspection
|
||||
///
|
||||
/// See NodeMime for recursive logic
|
||||
pub struct SelectedMime<'a>(pub &'a AnyPart<'a>);
|
||||
impl<'a> SelectedMime<'a> {
|
||||
pub fn header_value(&'a self, to_match_ext: &[u8]) -> Option<&'a [u8]> {
|
||||
let to_match = to_match_ext.to_ascii_lowercase();
|
||||
|
||||
self.eml_mime()
|
||||
.kv
|
||||
.iter()
|
||||
.filter_map(|field| match field {
|
||||
header::Field::Good(header::Kv2(k, v)) => Some((k, v)),
|
||||
_ => None,
|
||||
})
|
||||
.find(|(k, _)| k.to_ascii_lowercase() == to_match)
|
||||
.map(|(_, v)| v)
|
||||
.copied()
|
||||
}
|
||||
|
||||
/// The subsetted fetch section basically tells us the
|
||||
/// extraction logic to apply on our selected MIME.
|
||||
/// This function acts as a router for these logic.
|
||||
fn extract(&self, extractor: &SubsettedSection<'a>) -> Result<ExtractedFull<'a>> {
|
||||
match extractor {
|
||||
SubsettedSection::Text => self.text(),
|
||||
SubsettedSection::Header => self.header(),
|
||||
SubsettedSection::HeaderFields(fields) => self.header_fields(fields, false),
|
||||
SubsettedSection::HeaderFieldsNot(fields) => self.header_fields(fields, true),
|
||||
SubsettedSection::Part => self.part(),
|
||||
SubsettedSection::Mime => self.mime(),
|
||||
}
|
||||
}
|
||||
|
||||
fn mime(&self) -> Result<ExtractedFull<'a>> {
|
||||
let bytes = match &self.0 {
|
||||
AnyPart::Txt(p) => p.mime.fields.raw,
|
||||
AnyPart::Bin(p) => p.mime.fields.raw,
|
||||
AnyPart::Msg(p) => p.child.mime().raw,
|
||||
AnyPart::Mult(p) => p.mime.fields.raw,
|
||||
};
|
||||
Ok(ExtractedFull(bytes.into()))
|
||||
}
|
||||
|
||||
fn part(&self) -> Result<ExtractedFull<'a>> {
|
||||
let bytes = match &self.0 {
|
||||
AnyPart::Txt(p) => p.body,
|
||||
AnyPart::Bin(p) => p.body,
|
||||
AnyPart::Msg(p) => p.raw_part,
|
||||
AnyPart::Mult(_) => bail!("Multipart part has no body"),
|
||||
};
|
||||
Ok(ExtractedFull(bytes.to_vec().into()))
|
||||
}
|
||||
|
||||
fn eml_mime(&self) -> &eml_codec::mime::NaiveMIME<'_> {
|
||||
match &self.0 {
|
||||
AnyPart::Msg(msg) => msg.child.mime(),
|
||||
other => other.mime(),
|
||||
}
|
||||
}
|
||||
|
||||
/// The [...] HEADER.FIELDS, and HEADER.FIELDS.NOT part
|
||||
/// specifiers refer to the [RFC-2822] header of the message or of
|
||||
/// an encapsulated [MIME-IMT] MESSAGE/RFC822 message.
|
||||
/// HEADER.FIELDS and HEADER.FIELDS.NOT are followed by a list of
|
||||
/// field-name (as defined in [RFC-2822]) names, and return a
|
||||
/// subset of the header. The subset returned by HEADER.FIELDS
|
||||
/// contains only those header fields with a field-name that
|
||||
/// matches one of the names in the list; similarly, the subset
|
||||
/// returned by HEADER.FIELDS.NOT contains only the header fields
|
||||
/// with a non-matching field-name. The field-matching is
|
||||
/// case-insensitive but otherwise exact.
|
||||
fn header_fields(
|
||||
&self,
|
||||
fields: &'a Vec1<AString<'a>>,
|
||||
invert: bool,
|
||||
) -> Result<ExtractedFull<'a>> {
|
||||
// Build a lowercase ascii hashset with the fields to fetch
|
||||
let index = fields
|
||||
.as_ref()
|
||||
.iter()
|
||||
.map(|x| {
|
||||
match x {
|
||||
AString::Atom(a) => a.inner().as_bytes(),
|
||||
AString::String(IString::Literal(l)) => l.as_ref(),
|
||||
AString::String(IString::Quoted(q)) => q.inner().as_bytes(),
|
||||
}
|
||||
.to_ascii_lowercase()
|
||||
})
|
||||
.collect::<HashSet<_>>();
|
||||
|
||||
// Extract MIME headers
|
||||
let mime = self.eml_mime();
|
||||
|
||||
// Filter our MIME headers based on the field index
|
||||
// 1. Keep only the correctly formatted headers
|
||||
// 2. Keep only based on the index presence or absence
|
||||
// 3. Reduce as a byte vector
|
||||
let buffer = mime
|
||||
.kv
|
||||
.iter()
|
||||
.filter_map(|field| match field {
|
||||
header::Field::Good(header::Kv2(k, v)) => Some((k, v)),
|
||||
_ => None,
|
||||
})
|
||||
.filter(|(k, _)| index.contains(&k.to_ascii_lowercase()) ^ invert)
|
||||
.fold(vec![], |mut acc, (k, v)| {
|
||||
acc.extend(*k);
|
||||
acc.extend(b": ");
|
||||
acc.extend(*v);
|
||||
acc.extend(b"\r\n");
|
||||
acc
|
||||
});
|
||||
|
||||
Ok(ExtractedFull(buffer.into()))
|
||||
}
|
||||
|
||||
/// The HEADER [...] part specifiers refer to the [RFC-2822] header of the message or of
|
||||
/// an encapsulated [MIME-IMT] MESSAGE/RFC822 message.
|
||||
/// ```raw
|
||||
/// HEADER ([RFC-2822] header of the message)
|
||||
/// ```
|
||||
fn header(&self) -> Result<ExtractedFull<'a>> {
|
||||
let msg = self
|
||||
.0
|
||||
.as_message()
|
||||
.ok_or(anyhow!("Selected part must be a message/rfc822"))?;
|
||||
Ok(ExtractedFull(msg.raw_headers.into()))
|
||||
}
|
||||
|
||||
/// The TEXT part specifier refers to the text body of the message, omitting the [RFC-2822] header.
|
||||
fn text(&self) -> Result<ExtractedFull<'a>> {
|
||||
let msg = self
|
||||
.0
|
||||
.as_message()
|
||||
.ok_or(anyhow!("Selected part must be a message/rfc822"))?;
|
||||
Ok(ExtractedFull(msg.raw_body.into()))
|
||||
}
|
||||
|
||||
// ------------
|
||||
|
||||
/// Basic field of a MIME part that is
|
||||
/// common to all parts
|
||||
fn basic_fields(&self) -> Result<BasicFields<'static>> {
|
||||
let sz = match self.0 {
|
||||
AnyPart::Txt(x) => x.body.len(),
|
||||
AnyPart::Bin(x) => x.body.len(),
|
||||
AnyPart::Msg(x) => x.raw_part.len(),
|
||||
AnyPart::Mult(_) => 0,
|
||||
};
|
||||
let m = self.0.mime();
|
||||
let parameter_list = m
|
||||
.ctype
|
||||
.as_ref()
|
||||
.map(|x| {
|
||||
x.params
|
||||
.iter()
|
||||
.map(|p| {
|
||||
(
|
||||
IString::try_from(String::from_utf8_lossy(p.name).to_string()),
|
||||
IString::try_from(p.value.to_string()),
|
||||
)
|
||||
})
|
||||
.filter(|(k, v)| k.is_ok() && v.is_ok())
|
||||
.map(|(k, v)| (k.unwrap(), v.unwrap()))
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or(vec![]);
|
||||
|
||||
Ok(BasicFields {
|
||||
parameter_list,
|
||||
id: NString(
|
||||
m.id.as_ref()
|
||||
.and_then(|ci| IString::try_from(ci.to_string()).ok()),
|
||||
),
|
||||
description: NString(
|
||||
m.description
|
||||
.as_ref()
|
||||
.and_then(|cd| IString::try_from(cd.to_string()).ok()),
|
||||
),
|
||||
content_transfer_encoding: match m.transfer_encoding {
|
||||
mime::mechanism::Mechanism::_8Bit => unchecked_istring("8bit"),
|
||||
mime::mechanism::Mechanism::Binary => unchecked_istring("binary"),
|
||||
mime::mechanism::Mechanism::QuotedPrintable => {
|
||||
unchecked_istring("quoted-printable")
|
||||
}
|
||||
mime::mechanism::Mechanism::Base64 => unchecked_istring("base64"),
|
||||
_ => unchecked_istring("7bit"),
|
||||
},
|
||||
// @FIXME we can't compute the size of the message currently...
|
||||
size: u32::try_from(sz)?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------
|
||||
struct NodeMsg<'a>(&'a NodeMime<'a>, &'a composite::Message<'a>);
|
||||
impl<'a> NodeMsg<'a> {
|
||||
fn structure(&self, is_ext: bool) -> Result<BodyStructure<'static>> {
|
||||
let basic = SelectedMime(self.0 .0).basic_fields()?;
|
||||
|
||||
Ok(BodyStructure::Single {
|
||||
body: FetchBody {
|
||||
basic,
|
||||
specific: SpecificFields::Message {
|
||||
envelope: Box::new(ImfView(&self.1.imf).message_envelope()),
|
||||
body_structure: Box::new(NodeMime(&self.1.child).structure(is_ext)?),
|
||||
number_of_lines: nol(self.1.raw_part),
|
||||
},
|
||||
},
|
||||
extension_data: match is_ext {
|
||||
true => Some(SinglePartExtensionData {
|
||||
md5: NString(None),
|
||||
tail: None,
|
||||
}),
|
||||
_ => None,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
struct NodeMult<'a>(&'a NodeMime<'a>, &'a composite::Multipart<'a>);
|
||||
impl<'a> NodeMult<'a> {
|
||||
fn structure(&self, is_ext: bool) -> Result<BodyStructure<'static>> {
|
||||
let itype = &self.1.mime.interpreted_type;
|
||||
let subtype = IString::try_from(itype.subtype.to_string())
|
||||
.unwrap_or(unchecked_istring("alternative"));
|
||||
|
||||
let inner_bodies = self
|
||||
.1
|
||||
.children
|
||||
.iter()
|
||||
.filter_map(|inner| NodeMime(&inner).structure(is_ext).ok())
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
Vec1::validate(&inner_bodies)?;
|
||||
let bodies = Vec1::unvalidated(inner_bodies);
|
||||
|
||||
Ok(BodyStructure::Multi {
|
||||
bodies,
|
||||
subtype,
|
||||
extension_data: match is_ext {
|
||||
true => Some(MultiPartExtensionData {
|
||||
parameter_list: vec![(
|
||||
IString::try_from("boundary").unwrap(),
|
||||
IString::try_from(self.1.mime.interpreted_type.boundary.to_string())?,
|
||||
)],
|
||||
tail: None,
|
||||
}),
|
||||
_ => None,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
struct NodeTxt<'a>(&'a NodeMime<'a>, &'a discrete::Text<'a>);
|
||||
impl<'a> NodeTxt<'a> {
|
||||
fn structure(&self, is_ext: bool) -> Result<BodyStructure<'static>> {
|
||||
let mut basic = SelectedMime(self.0 .0).basic_fields()?;
|
||||
|
||||
// Get the interpreted content type, set it
|
||||
let itype = match &self.1.mime.interpreted_type {
|
||||
Deductible::Inferred(v) | Deductible::Explicit(v) => v,
|
||||
};
|
||||
let subtype =
|
||||
IString::try_from(itype.subtype.to_string()).unwrap_or(unchecked_istring("plain"));
|
||||
|
||||
// Add charset to the list of parameters if we know it has been inferred as it will be
|
||||
// missing from the parsed content.
|
||||
if let Deductible::Inferred(charset) = &itype.charset {
|
||||
basic.parameter_list.push((
|
||||
unchecked_istring("charset"),
|
||||
IString::try_from(charset.to_string()).unwrap_or(unchecked_istring("us-ascii")),
|
||||
));
|
||||
}
|
||||
|
||||
Ok(BodyStructure::Single {
|
||||
body: FetchBody {
|
||||
basic,
|
||||
specific: SpecificFields::Text {
|
||||
subtype,
|
||||
number_of_lines: nol(self.1.body),
|
||||
},
|
||||
},
|
||||
extension_data: match is_ext {
|
||||
true => Some(SinglePartExtensionData {
|
||||
md5: NString(None),
|
||||
tail: None,
|
||||
}),
|
||||
_ => None,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct NodeBin<'a>(&'a NodeMime<'a>, &'a discrete::Binary<'a>);
|
||||
impl<'a> NodeBin<'a> {
|
||||
fn structure(&self, is_ext: bool) -> Result<BodyStructure<'static>> {
|
||||
let basic = SelectedMime(self.0 .0).basic_fields()?;
|
||||
|
||||
let default = mime::r#type::NaiveType {
|
||||
main: &b"application"[..],
|
||||
sub: &b"octet-stream"[..],
|
||||
params: vec![],
|
||||
};
|
||||
let ct = self.1.mime.fields.ctype.as_ref().unwrap_or(&default);
|
||||
|
||||
let r#type = IString::try_from(String::from_utf8_lossy(ct.main).to_string()).or(Err(
|
||||
anyhow!("Unable to build IString from given Content-Type type given"),
|
||||
))?;
|
||||
|
||||
let subtype = IString::try_from(String::from_utf8_lossy(ct.sub).to_string()).or(Err(
|
||||
anyhow!("Unable to build IString from given Content-Type subtype given"),
|
||||
))?;
|
||||
|
||||
Ok(BodyStructure::Single {
|
||||
body: FetchBody {
|
||||
basic,
|
||||
specific: SpecificFields::Basic { r#type, subtype },
|
||||
},
|
||||
extension_data: match is_ext {
|
||||
true => Some(SinglePartExtensionData {
|
||||
md5: NString(None),
|
||||
tail: None,
|
||||
}),
|
||||
_ => None,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------
|
||||
|
||||
struct ExtractedFull<'a>(Cow<'a, [u8]>);
|
||||
impl<'a> ExtractedFull<'a> {
|
||||
/// It is possible to fetch a substring of the designated text.
|
||||
/// This is done by appending an open angle bracket ("<"), the
|
||||
/// octet position of the first desired octet, a period, the
|
||||
/// maximum number of octets desired, and a close angle bracket
|
||||
/// (">") to the part specifier. If the starting octet is beyond
|
||||
/// the end of the text, an empty string is returned.
|
||||
///
|
||||
/// Any partial fetch that attempts to read beyond the end of the
|
||||
/// text is truncated as appropriate. A partial fetch that starts
|
||||
/// at octet 0 is returned as a partial fetch, even if this
|
||||
/// truncation happened.
|
||||
///
|
||||
/// Note: This means that BODY[]<0.2048> of a 1500-octet message
|
||||
/// will return BODY[]<0> with a literal of size 1500, not
|
||||
/// BODY[].
|
||||
///
|
||||
/// Note: A substring fetch of a HEADER.FIELDS or
|
||||
/// HEADER.FIELDS.NOT part specifier is calculated after
|
||||
/// subsetting the header.
|
||||
fn to_body_section(self, partial: &'_ Option<(u32, NonZeroU32)>) -> BodySection<'a> {
|
||||
match partial {
|
||||
Some((begin, len)) => self.partialize(*begin, *len),
|
||||
None => BodySection::Full(self.0),
|
||||
}
|
||||
}
|
||||
|
||||
fn partialize(self, begin: u32, len: NonZeroU32) -> BodySection<'a> {
|
||||
// Asked range is starting after the end of the content,
|
||||
// returning an empty buffer
|
||||
if begin as usize > self.0.len() {
|
||||
return BodySection::Slice {
|
||||
body: Cow::Borrowed(&[][..]),
|
||||
origin_octet: begin,
|
||||
};
|
||||
}
|
||||
|
||||
// Asked range is ending after the end of the content,
|
||||
// slice only the beginning of the buffer
|
||||
if (begin + len.get()) as usize >= self.0.len() {
|
||||
return BodySection::Slice {
|
||||
body: match self.0 {
|
||||
Cow::Borrowed(body) => Cow::Borrowed(&body[begin as usize..]),
|
||||
Cow::Owned(body) => Cow::Owned(body[begin as usize..].to_vec()),
|
||||
},
|
||||
origin_octet: begin,
|
||||
};
|
||||
}
|
||||
|
||||
// Range is included inside the considered content,
|
||||
// this is the "happy case"
|
||||
BodySection::Slice {
|
||||
body: match self.0 {
|
||||
Cow::Borrowed(body) => {
|
||||
Cow::Borrowed(&body[begin as usize..(begin + len.get()) as usize])
|
||||
}
|
||||
Cow::Owned(body) => {
|
||||
Cow::Owned(body[begin as usize..(begin + len.get()) as usize].to_vec())
|
||||
}
|
||||
},
|
||||
origin_octet: begin,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// ---- LEGACY
|
||||
|
||||
/// s is set to static to ensure that only compile time values
|
||||
/// checked by developpers are passed.
|
||||
fn unchecked_istring(s: &'static str) -> IString {
|
||||
IString::try_from(s).expect("this value is expected to be a valid imap-codec::IString")
|
||||
}
|
||||
|
||||
// Number Of Lines
|
||||
fn nol(input: &[u8]) -> u32 {
|
||||
input
|
||||
.iter()
|
||||
.filter(|x| **x == b'\n')
|
||||
.count()
|
||||
.try_into()
|
||||
.unwrap_or(0)
|
||||
}
|
||||
471
src/imap/mod.rs
471
src/imap/mod.rs
|
|
@ -1,421 +1,98 @@
|
|||
mod attributes;
|
||||
mod capability;
|
||||
mod command;
|
||||
mod flags;
|
||||
mod flow;
|
||||
mod imf_view;
|
||||
mod index;
|
||||
mod mail_view;
|
||||
mod mailbox_view;
|
||||
mod mime_view;
|
||||
mod request;
|
||||
mod response;
|
||||
mod search;
|
||||
mod session;
|
||||
|
||||
use std::net::SocketAddr;
|
||||
use std::task::{Context, Poll};
|
||||
|
||||
use anyhow::{anyhow, bail, Context, Result};
|
||||
use futures::stream::{FuturesUnordered, StreamExt};
|
||||
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::sync::mpsc;
|
||||
use anyhow::Result;
|
||||
use boitalettres::errors::Error as BalError;
|
||||
use boitalettres::proto::{Request, Response};
|
||||
use boitalettres::server::accept::addr::AddrIncoming;
|
||||
use boitalettres::server::accept::addr::AddrStream;
|
||||
use boitalettres::server::Server as ImapServer;
|
||||
use futures::future::BoxFuture;
|
||||
use futures::future::FutureExt;
|
||||
use tokio::sync::watch;
|
||||
use tower::Service;
|
||||
|
||||
use imap_codec::imap_types::response::{Code, CommandContinuationRequest, Response, Status};
|
||||
use imap_codec::imap_types::{core::Text, response::Greeting};
|
||||
use imap_flow::server::{ServerFlow, ServerFlowEvent, ServerFlowOptions};
|
||||
use imap_flow::stream::AnyStream;
|
||||
use rustls_pemfile::{certs, private_key};
|
||||
use tokio_rustls::TlsAcceptor;
|
||||
|
||||
use crate::config::{ImapConfig, ImapUnsecureConfig};
|
||||
use crate::imap::capability::ServerCapability;
|
||||
use crate::imap::request::Request;
|
||||
use crate::imap::response::{Body, ResponseOrIdle};
|
||||
use crate::imap::session::Instance;
|
||||
use crate::config::ImapConfig;
|
||||
use crate::login::ArcLoginProvider;
|
||||
|
||||
/// Server is a thin wrapper to register our Services in BàL
|
||||
pub struct Server {
|
||||
bind_addr: SocketAddr,
|
||||
login_provider: ArcLoginProvider,
|
||||
capabilities: ServerCapability,
|
||||
tls: Option<TlsAcceptor>,
|
||||
pub struct Server(ImapServer<AddrIncoming, Instance>);
|
||||
pub async fn new(config: ImapConfig, login: ArcLoginProvider) -> Result<Server> {
|
||||
//@FIXME add a configuration parameter
|
||||
let incoming = AddrIncoming::new(config.bind_addr).await?;
|
||||
tracing::info!("IMAP activated, will listen on {:#}", incoming.local_addr);
|
||||
|
||||
let imap = ImapServer::new(incoming).serve(Instance::new(login.clone()));
|
||||
Ok(Server(imap))
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ClientContext {
|
||||
addr: SocketAddr,
|
||||
login_provider: ArcLoginProvider,
|
||||
must_exit: watch::Receiver<bool>,
|
||||
server_capabilities: ServerCapability,
|
||||
}
|
||||
|
||||
pub fn new(config: ImapConfig, login: ArcLoginProvider) -> Result<Server> {
|
||||
let loaded_certs = certs(&mut std::io::BufReader::new(std::fs::File::open(
|
||||
config.certs,
|
||||
)?))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
let loaded_key = private_key(&mut std::io::BufReader::new(std::fs::File::open(
|
||||
config.key,
|
||||
)?))?
|
||||
.unwrap();
|
||||
|
||||
let tls_config = rustls::ServerConfig::builder()
|
||||
.with_no_client_auth()
|
||||
.with_single_cert(loaded_certs, loaded_key)?;
|
||||
let acceptor = TlsAcceptor::from(Arc::new(tls_config));
|
||||
|
||||
Ok(Server {
|
||||
bind_addr: config.bind_addr,
|
||||
login_provider: login,
|
||||
capabilities: ServerCapability::default(),
|
||||
tls: Some(acceptor),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn new_unsecure(config: ImapUnsecureConfig, login: ArcLoginProvider) -> Server {
|
||||
Server {
|
||||
bind_addr: config.bind_addr,
|
||||
login_provider: login,
|
||||
capabilities: ServerCapability::default(),
|
||||
tls: None,
|
||||
}
|
||||
}
|
||||
|
||||
impl Server {
|
||||
pub async fn run(self: Self, mut must_exit: watch::Receiver<bool>) -> Result<()> {
|
||||
let tcp = TcpListener::bind(self.bind_addr).await?;
|
||||
tracing::info!("IMAP server listening on {:#}", self.bind_addr);
|
||||
|
||||
let mut connections = FuturesUnordered::new();
|
||||
|
||||
while !*must_exit.borrow() {
|
||||
let wait_conn_finished = async {
|
||||
if connections.is_empty() {
|
||||
futures::future::pending().await
|
||||
} else {
|
||||
connections.next().await
|
||||
}
|
||||
};
|
||||
let (socket, remote_addr) = tokio::select! {
|
||||
a = tcp.accept() => a?,
|
||||
_ = wait_conn_finished => continue,
|
||||
_ = must_exit.changed() => continue,
|
||||
};
|
||||
tracing::info!("IMAP: accepted connection from {}", remote_addr);
|
||||
let stream = match self.tls.clone() {
|
||||
Some(acceptor) => {
|
||||
let stream = match acceptor.accept(socket).await {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
tracing::error!(err=?e, "TLS negociation failed");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
AnyStream::new(stream)
|
||||
}
|
||||
None => AnyStream::new(socket),
|
||||
};
|
||||
|
||||
let client = ClientContext {
|
||||
addr: remote_addr.clone(),
|
||||
login_provider: self.login_provider.clone(),
|
||||
must_exit: must_exit.clone(),
|
||||
server_capabilities: self.capabilities.clone(),
|
||||
};
|
||||
let conn = tokio::spawn(NetLoop::handler(client, stream));
|
||||
connections.push(conn);
|
||||
pub async fn run(self, mut must_exit: watch::Receiver<bool>) -> Result<()> {
|
||||
tracing::info!("IMAP started!");
|
||||
tokio::select! {
|
||||
s = self.0 => s?,
|
||||
_ = must_exit.changed() => tracing::info!("Stopped IMAP server"),
|
||||
}
|
||||
drop(tcp);
|
||||
|
||||
tracing::info!("IMAP server shutting down, draining remaining connections...");
|
||||
while connections.next().await.is_some() {}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::mpsc::*;
|
||||
use tokio::sync::Notify;
|
||||
use tokio_util::bytes::BytesMut;
|
||||
//---
|
||||
|
||||
const PIPELINABLE_COMMANDS: usize = 64;
|
||||
/// Instance is the main Tokio Tower service that we register in BàL.
|
||||
/// It receives new connection demands and spawn a dedicated service.
|
||||
struct Instance {
|
||||
login_provider: ArcLoginProvider,
|
||||
}
|
||||
impl Instance {
|
||||
pub fn new(login_provider: ArcLoginProvider) -> Self {
|
||||
Self { login_provider }
|
||||
}
|
||||
}
|
||||
impl<'a> Service<&'a AddrStream> for Instance {
|
||||
type Response = Connection;
|
||||
type Error = anyhow::Error;
|
||||
type Future = BoxFuture<'static, Result<Self::Response>>;
|
||||
|
||||
// @FIXME a full refactor of this part of the code will be needed sooner or later
|
||||
struct NetLoop {
|
||||
ctx: ClientContext,
|
||||
server: ServerFlow,
|
||||
cmd_tx: Sender<Request>,
|
||||
resp_rx: UnboundedReceiver<ResponseOrIdle>,
|
||||
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn call(&mut self, addr: &'a AddrStream) -> Self::Future {
|
||||
tracing::info!(remote_addr = %addr.remote_addr, local_addr = %addr.local_addr, "accept");
|
||||
let lp = self.login_provider.clone();
|
||||
async { Ok(Connection::new(lp)) }.boxed()
|
||||
}
|
||||
}
|
||||
|
||||
impl NetLoop {
|
||||
async fn handler(ctx: ClientContext, sock: AnyStream) {
|
||||
let addr = ctx.addr.clone();
|
||||
//---
|
||||
|
||||
let mut nl = match Self::new(ctx, sock).await {
|
||||
Ok(nl) => {
|
||||
tracing::debug!(addr=?addr, "netloop successfully initialized");
|
||||
nl
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!(addr=?addr, err=?e, "netloop can not be initialized, closing session");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
match nl.core().await {
|
||||
Ok(()) => {
|
||||
tracing::debug!("closing successful netloop core for {:?}", addr);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!("closing errored netloop core for {:?}: {}", addr, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn new(ctx: ClientContext, sock: AnyStream) -> Result<Self> {
|
||||
let mut opts = ServerFlowOptions::default();
|
||||
opts.crlf_relaxed = false;
|
||||
opts.literal_accept_text = Text::unvalidated("OK");
|
||||
opts.literal_reject_text = Text::unvalidated("Literal rejected");
|
||||
|
||||
// Send greeting
|
||||
let (server, _) = ServerFlow::send_greeting(
|
||||
sock,
|
||||
opts,
|
||||
Greeting::ok(
|
||||
Some(Code::Capability(ctx.server_capabilities.to_vec())),
|
||||
"Aerogramme",
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Start a mailbox session in background
|
||||
let (cmd_tx, cmd_rx) = mpsc::channel::<Request>(PIPELINABLE_COMMANDS);
|
||||
let (resp_tx, resp_rx) = mpsc::unbounded_channel::<ResponseOrIdle>();
|
||||
tokio::spawn(Self::session(ctx.clone(), cmd_rx, resp_tx));
|
||||
|
||||
// Return the object
|
||||
Ok(NetLoop {
|
||||
ctx,
|
||||
server,
|
||||
cmd_tx,
|
||||
resp_rx,
|
||||
})
|
||||
}
|
||||
|
||||
/// Coms with the background session
|
||||
async fn session(
|
||||
ctx: ClientContext,
|
||||
mut cmd_rx: Receiver<Request>,
|
||||
resp_tx: UnboundedSender<ResponseOrIdle>,
|
||||
) -> () {
|
||||
let mut session = Instance::new(ctx.login_provider, ctx.server_capabilities);
|
||||
loop {
|
||||
let cmd = match cmd_rx.recv().await {
|
||||
None => break,
|
||||
Some(cmd_recv) => cmd_recv,
|
||||
};
|
||||
|
||||
tracing::debug!(cmd=?cmd, sock=%ctx.addr, "command");
|
||||
let maybe_response = session.request(cmd).await;
|
||||
tracing::debug!(cmd=?maybe_response, sock=%ctx.addr, "response");
|
||||
|
||||
match resp_tx.send(maybe_response) {
|
||||
Err(_) => break,
|
||||
Ok(_) => (),
|
||||
};
|
||||
}
|
||||
tracing::info!("runner is quitting");
|
||||
}
|
||||
|
||||
async fn core(&mut self) -> Result<()> {
|
||||
let mut maybe_idle: Option<Arc<Notify>> = None;
|
||||
loop {
|
||||
tokio::select! {
|
||||
// Managing imap_flow stuff
|
||||
srv_evt = self.server.progress() => match srv_evt? {
|
||||
ServerFlowEvent::ResponseSent { handle: _handle, response } => {
|
||||
match response {
|
||||
Response::Status(Status::Bye(_)) => return Ok(()),
|
||||
_ => tracing::trace!("sent to {} content {:?}", self.ctx.addr, response),
|
||||
}
|
||||
},
|
||||
ServerFlowEvent::CommandReceived { command } => {
|
||||
match self.cmd_tx.try_send(Request::ImapCommand(command)) {
|
||||
Ok(_) => (),
|
||||
Err(mpsc::error::TrySendError::Full(_)) => {
|
||||
self.server.enqueue_status(Status::bye(None, "Too fast").unwrap());
|
||||
tracing::error!("client {:?} is sending commands too fast, closing.", self.ctx.addr);
|
||||
}
|
||||
_ => {
|
||||
self.server.enqueue_status(Status::bye(None, "Internal session exited").unwrap());
|
||||
tracing::error!("session task exited for {:?}, quitting", self.ctx.addr);
|
||||
}
|
||||
}
|
||||
},
|
||||
ServerFlowEvent::IdleCommandReceived { tag } => {
|
||||
match self.cmd_tx.try_send(Request::IdleStart(tag)) {
|
||||
Ok(_) => (),
|
||||
Err(mpsc::error::TrySendError::Full(_)) => {
|
||||
self.server.enqueue_status(Status::bye(None, "Too fast").unwrap());
|
||||
tracing::error!("client {:?} is sending commands too fast, closing.", self.ctx.addr);
|
||||
}
|
||||
_ => {
|
||||
self.server.enqueue_status(Status::bye(None, "Internal session exited").unwrap());
|
||||
tracing::error!("session task exited for {:?}, quitting", self.ctx.addr);
|
||||
}
|
||||
}
|
||||
}
|
||||
ServerFlowEvent::IdleDoneReceived => {
|
||||
tracing::trace!("client sent DONE and want to stop IDLE");
|
||||
maybe_idle.ok_or(anyhow!("Received IDLE done but not idling currently"))?.notify_one();
|
||||
maybe_idle = None;
|
||||
}
|
||||
flow => {
|
||||
self.server.enqueue_status(Status::bye(None, "Unsupported server flow event").unwrap());
|
||||
tracing::error!("session task exited for {:?} due to unsupported flow {:?}", self.ctx.addr, flow);
|
||||
}
|
||||
},
|
||||
|
||||
// Managing response generated by Aerogramme
|
||||
maybe_msg = self.resp_rx.recv() => match maybe_msg {
|
||||
Some(ResponseOrIdle::Response(response)) => {
|
||||
tracing::trace!("Interactive, server has a response for the client");
|
||||
for body_elem in response.body.into_iter() {
|
||||
let _handle = match body_elem {
|
||||
Body::Data(d) => self.server.enqueue_data(d),
|
||||
Body::Status(s) => self.server.enqueue_status(s),
|
||||
};
|
||||
}
|
||||
self.server.enqueue_status(response.completion);
|
||||
},
|
||||
Some(ResponseOrIdle::IdleAccept(stop)) => {
|
||||
tracing::trace!("Interactive, server agreed to switch in idle mode");
|
||||
let cr = CommandContinuationRequest::basic(None, "Idling")?;
|
||||
self.server.idle_accept(cr).or(Err(anyhow!("refused continuation for idle accept")))?;
|
||||
self.cmd_tx.try_send(Request::IdlePoll)?;
|
||||
if maybe_idle.is_some() {
|
||||
bail!("Can't start IDLE if already idling");
|
||||
}
|
||||
maybe_idle = Some(stop);
|
||||
},
|
||||
Some(ResponseOrIdle::IdleEvent(elems)) => {
|
||||
tracing::trace!("server imap session has some change to communicate to the client");
|
||||
for body_elem in elems.into_iter() {
|
||||
let _handle = match body_elem {
|
||||
Body::Data(d) => self.server.enqueue_data(d),
|
||||
Body::Status(s) => self.server.enqueue_status(s),
|
||||
};
|
||||
}
|
||||
self.cmd_tx.try_send(Request::IdlePoll)?;
|
||||
},
|
||||
Some(ResponseOrIdle::IdleReject(response)) => {
|
||||
tracing::trace!("inform client that session rejected idle");
|
||||
self.server
|
||||
.idle_reject(response.completion)
|
||||
.or(Err(anyhow!("wrong reject command")))?;
|
||||
},
|
||||
None => {
|
||||
self.server.enqueue_status(Status::bye(None, "Internal session exited").unwrap());
|
||||
tracing::error!("session task exited for {:?}, quitting", self.ctx.addr);
|
||||
},
|
||||
Some(_) => unreachable!(),
|
||||
|
||||
},
|
||||
|
||||
// When receiving a CTRL+C
|
||||
_ = self.ctx.must_exit.changed() => {
|
||||
tracing::trace!("Interactive, CTRL+C, exiting");
|
||||
self.server.enqueue_status(Status::bye(None, "Server is being shutdown").unwrap());
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
async fn idle_mode(&mut self, mut buff: BytesMut, stop: Arc<Notify>) -> Result<LoopMode> {
|
||||
// Flush send
|
||||
loop {
|
||||
tracing::trace!("flush server send");
|
||||
match self.server.progress_send().await? {
|
||||
Some(..) => continue,
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
|
||||
tokio::select! {
|
||||
// Receiving IDLE event from background
|
||||
maybe_msg = self.resp_rx.recv() => match maybe_msg {
|
||||
// Session decided idle is terminated
|
||||
Some(ResponseOrIdle::Response(response)) => {
|
||||
tracing::trace!("server imap session said idle is done, sending response done, switching to interactive");
|
||||
for body_elem in response.body.into_iter() {
|
||||
let _handle = match body_elem {
|
||||
Body::Data(d) => self.server.enqueue_data(d),
|
||||
Body::Status(s) => self.server.enqueue_status(s),
|
||||
};
|
||||
}
|
||||
self.server.enqueue_status(response.completion);
|
||||
return Ok(LoopMode::Interactive)
|
||||
},
|
||||
// Session has some information for user
|
||||
Some(ResponseOrIdle::IdleEvent(elems)) => {
|
||||
tracing::trace!("server imap session has some change to communicate to the client");
|
||||
for body_elem in elems.into_iter() {
|
||||
let _handle = match body_elem {
|
||||
Body::Data(d) => self.server.enqueue_data(d),
|
||||
Body::Status(s) => self.server.enqueue_status(s),
|
||||
};
|
||||
}
|
||||
self.cmd_tx.try_send(Request::Idle)?;
|
||||
return Ok(LoopMode::Idle(buff, stop))
|
||||
},
|
||||
|
||||
// Session crashed
|
||||
None => {
|
||||
self.server.enqueue_status(Status::bye(None, "Internal session exited").unwrap());
|
||||
tracing::error!("session task exited for {:?}, quitting", self.ctx.addr);
|
||||
return Ok(LoopMode::Interactive)
|
||||
},
|
||||
|
||||
// Session can't start idling while already idling, it's a logic error!
|
||||
Some(ResponseOrIdle::StartIdle(..)) => bail!("can't start idling while already idling!"),
|
||||
},
|
||||
|
||||
// User is trying to interact with us
|
||||
read_client_result = self.server.stream.read(&mut buff) => {
|
||||
let _bytes_read = read_client_result?;
|
||||
use imap_codec::decode::Decoder;
|
||||
let codec = imap_codec::IdleDoneCodec::new();
|
||||
tracing::trace!("client sent some data for the server IMAP session");
|
||||
match codec.decode(&buff) {
|
||||
Ok(([], imap_codec::imap_types::extensions::idle::IdleDone)) => {
|
||||
// Session will be informed that it must stop idle
|
||||
// It will generate the "done" message and change the loop mode
|
||||
tracing::trace!("client sent DONE and want to stop IDLE");
|
||||
stop.notify_one()
|
||||
},
|
||||
Err(_) => {
|
||||
tracing::trace!("Unable to decode DONE, maybe not enough data were sent?");
|
||||
},
|
||||
_ => bail!("Client sent data after terminating the continuation without waiting for the server. This is an unsupported behavior and bug in Aerogramme, quitting."),
|
||||
};
|
||||
|
||||
return Ok(LoopMode::Idle(buff, stop))
|
||||
},
|
||||
|
||||
// When receiving a CTRL+C
|
||||
_ = self.ctx.must_exit.changed() => {
|
||||
tracing::trace!("CTRL+C sent, aborting IDLE for this session");
|
||||
self.server.enqueue_status(Status::bye(None, "Server is being shutdown").unwrap());
|
||||
return Ok(LoopMode::Interactive)
|
||||
},
|
||||
};
|
||||
}*/
|
||||
/// Connection is the per-connection Tokio Tower service we register in BàL.
|
||||
/// It handles a single TCP connection, and thus has a business logic.
|
||||
struct Connection {
|
||||
session: session::Manager,
|
||||
}
|
||||
impl Connection {
|
||||
pub fn new(login_provider: ArcLoginProvider) -> Self {
|
||||
Self {
|
||||
session: session::Manager::new(login_provider),
|
||||
}
|
||||
}
|
||||
}
|
||||
impl Service<Request> for Connection {
|
||||
type Response = Response;
|
||||
type Error = BalError;
|
||||
type Future = BoxFuture<'static, Result<Self::Response, Self::Error>>;
|
||||
|
||||
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn call(&mut self, req: Request) -> Self::Future {
|
||||
tracing::debug!("Got request: {:#?}", req.command);
|
||||
self.session.process(req)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +0,0 @@
|
|||
use imap_codec::imap_types::command::Command;
|
||||
use imap_codec::imap_types::core::Tag;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum Request {
|
||||
ImapCommand(Command<'static>),
|
||||
IdleStart(Tag<'static>),
|
||||
IdlePoll,
|
||||
}
|
||||
|
|
@ -1,124 +0,0 @@
|
|||
use anyhow::Result;
|
||||
use imap_codec::imap_types::command::Command;
|
||||
use imap_codec::imap_types::core::Tag;
|
||||
use imap_codec::imap_types::response::{Code, Data, Status};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::Notify;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum Body<'a> {
|
||||
Data(Data<'a>),
|
||||
Status(Status<'a>),
|
||||
}
|
||||
|
||||
pub struct ResponseBuilder<'a> {
|
||||
tag: Option<Tag<'a>>,
|
||||
code: Option<Code<'a>>,
|
||||
text: String,
|
||||
body: Vec<Body<'a>>,
|
||||
}
|
||||
|
||||
impl<'a> ResponseBuilder<'a> {
|
||||
pub fn to_req(mut self, cmd: &Command<'a>) -> Self {
|
||||
self.tag = Some(cmd.tag.clone());
|
||||
self
|
||||
}
|
||||
pub fn tag(mut self, tag: Tag<'a>) -> Self {
|
||||
self.tag = Some(tag);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn message(mut self, txt: impl Into<String>) -> Self {
|
||||
self.text = txt.into();
|
||||
self
|
||||
}
|
||||
|
||||
pub fn code(mut self, code: Code<'a>) -> Self {
|
||||
self.code = Some(code);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn data(mut self, data: Data<'a>) -> Self {
|
||||
self.body.push(Body::Data(data));
|
||||
self
|
||||
}
|
||||
|
||||
pub fn many_data(mut self, data: Vec<Data<'a>>) -> Self {
|
||||
for d in data.into_iter() {
|
||||
self = self.data(d);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub fn info(mut self, status: Status<'a>) -> Self {
|
||||
self.body.push(Body::Status(status));
|
||||
self
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub fn many_info(mut self, status: Vec<Status<'a>>) -> Self {
|
||||
for d in status.into_iter() {
|
||||
self = self.info(d);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
pub fn set_body(mut self, body: Vec<Body<'a>>) -> Self {
|
||||
self.body = body;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn ok(self) -> Result<Response<'a>> {
|
||||
Ok(Response {
|
||||
completion: Status::ok(self.tag, self.code, self.text)?,
|
||||
body: self.body,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn no(self) -> Result<Response<'a>> {
|
||||
Ok(Response {
|
||||
completion: Status::no(self.tag, self.code, self.text)?,
|
||||
body: self.body,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn bad(self) -> Result<Response<'a>> {
|
||||
Ok(Response {
|
||||
completion: Status::bad(self.tag, self.code, self.text)?,
|
||||
body: self.body,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Response<'a> {
|
||||
pub body: Vec<Body<'a>>,
|
||||
pub completion: Status<'a>,
|
||||
}
|
||||
|
||||
impl<'a> Response<'a> {
|
||||
pub fn build() -> ResponseBuilder<'a> {
|
||||
ResponseBuilder {
|
||||
tag: None,
|
||||
code: None,
|
||||
text: "".to_string(),
|
||||
body: vec![],
|
||||
}
|
||||
}
|
||||
|
||||
pub fn bye() -> Result<Response<'a>> {
|
||||
Ok(Response {
|
||||
completion: Status::bye(None, "bye")?,
|
||||
body: vec![],
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum ResponseOrIdle {
|
||||
Response(Response<'static>),
|
||||
IdleAccept(Arc<Notify>),
|
||||
IdleReject(Response<'static>),
|
||||
IdleEvent(Vec<Body<'static>>),
|
||||
}
|
||||
|
|
@ -1,477 +0,0 @@
|
|||
use std::num::{NonZeroU32, NonZeroU64};
|
||||
|
||||
use imap_codec::imap_types::core::Vec1;
|
||||
use imap_codec::imap_types::search::{MetadataItemSearch, SearchKey};
|
||||
use imap_codec::imap_types::sequence::{SeqOrUid, Sequence, SequenceSet};
|
||||
|
||||
use crate::imap::index::MailIndex;
|
||||
use crate::imap::mail_view::MailView;
|
||||
use crate::mail::query::QueryScope;
|
||||
|
||||
pub enum SeqType {
|
||||
Undefined,
|
||||
NonUid,
|
||||
Uid,
|
||||
}
|
||||
impl SeqType {
|
||||
pub fn is_uid(&self) -> bool {
|
||||
matches!(self, Self::Uid)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct Criteria<'a>(pub &'a SearchKey<'a>);
|
||||
impl<'a> Criteria<'a> {
|
||||
/// Returns a set of email identifiers that is greater or equal
|
||||
/// to the set of emails to return
|
||||
pub fn to_sequence_set(&self) -> (SequenceSet, SeqType) {
|
||||
match self.0 {
|
||||
SearchKey::All => (sequence_set_all(), SeqType::Undefined),
|
||||
SearchKey::SequenceSet(seq_set) => (seq_set.clone(), SeqType::NonUid),
|
||||
SearchKey::Uid(seq_set) => (seq_set.clone(), SeqType::Uid),
|
||||
SearchKey::Not(_inner) => {
|
||||
tracing::debug!(
|
||||
"using NOT in a search request is slow: it selects all identifiers"
|
||||
);
|
||||
(sequence_set_all(), SeqType::Undefined)
|
||||
}
|
||||
SearchKey::Or(left, right) => {
|
||||
tracing::debug!("using OR in a search request is slow: no deduplication is done");
|
||||
let (base, base_seqtype) = Self(&left).to_sequence_set();
|
||||
let (ext, ext_seqtype) = Self(&right).to_sequence_set();
|
||||
|
||||
// Check if we have a UID/ID conflict in fetching: now we don't know how to handle them
|
||||
match (base_seqtype, ext_seqtype) {
|
||||
(SeqType::Uid, SeqType::NonUid) | (SeqType::NonUid, SeqType::Uid) => {
|
||||
(sequence_set_all(), SeqType::Undefined)
|
||||
}
|
||||
(SeqType::Undefined, x) | (x, _) => {
|
||||
let mut new_vec = base.0.into_inner();
|
||||
new_vec.extend_from_slice(ext.0.as_ref());
|
||||
let seq = SequenceSet(
|
||||
Vec1::try_from(new_vec)
|
||||
.expect("merging non empty vec lead to non empty vec"),
|
||||
);
|
||||
(seq, x)
|
||||
}
|
||||
}
|
||||
}
|
||||
SearchKey::And(search_list) => {
|
||||
tracing::debug!(
|
||||
"using AND in a search request is slow: no intersection is performed"
|
||||
);
|
||||
// As we perform no intersection, we don't care if we mix uid or id.
|
||||
// We only keep the smallest range, being it ID or UID, depending of
|
||||
// which one has the less items. This is an approximation as UID ranges
|
||||
// can have holes while ID ones can't.
|
||||
search_list
|
||||
.as_ref()
|
||||
.iter()
|
||||
.map(|crit| Self(&crit).to_sequence_set())
|
||||
.min_by(|(x, _), (y, _)| {
|
||||
let x_size = approx_sequence_set_size(x);
|
||||
let y_size = approx_sequence_set_size(y);
|
||||
x_size.cmp(&y_size)
|
||||
})
|
||||
.unwrap_or((sequence_set_all(), SeqType::Undefined))
|
||||
}
|
||||
_ => (sequence_set_all(), SeqType::Undefined),
|
||||
}
|
||||
}
|
||||
|
||||
/// Not really clever as we can have cases where we filter out
|
||||
/// the email before needing to inspect its meta.
|
||||
/// But for now we are seeking the most basic/stupid algorithm.
|
||||
pub fn query_scope(&self) -> QueryScope {
|
||||
use SearchKey::*;
|
||||
match self.0 {
|
||||
// Combinators
|
||||
And(and_list) => and_list
|
||||
.as_ref()
|
||||
.iter()
|
||||
.fold(QueryScope::Index, |prev, sk| {
|
||||
prev.union(&Criteria(sk).query_scope())
|
||||
}),
|
||||
Not(inner) => Criteria(inner).query_scope(),
|
||||
Or(left, right) => Criteria(left)
|
||||
.query_scope()
|
||||
.union(&Criteria(right).query_scope()),
|
||||
All => QueryScope::Index,
|
||||
|
||||
// IMF Headers
|
||||
Bcc(_) | Cc(_) | From(_) | Header(..) | SentBefore(_) | SentOn(_) | SentSince(_)
|
||||
| Subject(_) | To(_) => QueryScope::Partial,
|
||||
// Internal Date is also stored in MailMeta
|
||||
Before(_) | On(_) | Since(_) => QueryScope::Partial,
|
||||
// Message size is also stored in MailMeta
|
||||
Larger(_) | Smaller(_) => QueryScope::Partial,
|
||||
// Text and Body require that we fetch the full content!
|
||||
Text(_) | Body(_) => QueryScope::Full,
|
||||
|
||||
_ => QueryScope::Index,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_modseq(&self) -> bool {
|
||||
use SearchKey::*;
|
||||
match self.0 {
|
||||
And(and_list) => and_list
|
||||
.as_ref()
|
||||
.iter()
|
||||
.any(|child| Criteria(child).is_modseq()),
|
||||
Or(left, right) => Criteria(left).is_modseq() || Criteria(right).is_modseq(),
|
||||
Not(child) => Criteria(child).is_modseq(),
|
||||
ModSeq { .. } => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns emails that we now for sure we want to keep
|
||||
/// but also a second list of emails we need to investigate further by
|
||||
/// fetching some remote data
|
||||
pub fn filter_on_idx<'b>(
|
||||
&self,
|
||||
midx_list: &[&'b MailIndex<'b>],
|
||||
) -> (Vec<&'b MailIndex<'b>>, Vec<&'b MailIndex<'b>>) {
|
||||
let (p1, p2): (Vec<_>, Vec<_>) = midx_list
|
||||
.iter()
|
||||
.map(|x| (x, self.is_keep_on_idx(x)))
|
||||
.filter(|(_midx, decision)| decision.is_keep())
|
||||
.map(|(midx, decision)| (*midx, decision))
|
||||
.partition(|(_midx, decision)| matches!(decision, PartialDecision::Keep));
|
||||
|
||||
let to_keep = p1.into_iter().map(|(v, _)| v).collect();
|
||||
let to_fetch = p2.into_iter().map(|(v, _)| v).collect();
|
||||
(to_keep, to_fetch)
|
||||
}
|
||||
|
||||
// ----
|
||||
|
||||
/// Here we are doing a partial filtering: we do not have access
|
||||
/// to the headers or to the body, so every time we encounter a rule
|
||||
/// based on them, we need to keep it.
|
||||
///
|
||||
/// @TODO Could be optimized on a per-email basis by also returning the QueryScope
|
||||
/// when more information is needed!
|
||||
fn is_keep_on_idx(&self, midx: &MailIndex) -> PartialDecision {
|
||||
use SearchKey::*;
|
||||
match self.0 {
|
||||
// Combinator logic
|
||||
And(expr_list) => expr_list
|
||||
.as_ref()
|
||||
.iter()
|
||||
.fold(PartialDecision::Keep, |acc, cur| {
|
||||
acc.and(&Criteria(cur).is_keep_on_idx(midx))
|
||||
}),
|
||||
Or(left, right) => {
|
||||
let left_decision = Criteria(left).is_keep_on_idx(midx);
|
||||
let right_decision = Criteria(right).is_keep_on_idx(midx);
|
||||
left_decision.or(&right_decision)
|
||||
}
|
||||
Not(expr) => Criteria(expr).is_keep_on_idx(midx).not(),
|
||||
All => PartialDecision::Keep,
|
||||
|
||||
// Sequence logic
|
||||
maybe_seq if is_sk_seq(maybe_seq) => is_keep_seq(maybe_seq, midx).into(),
|
||||
maybe_flag if is_sk_flag(maybe_flag) => is_keep_flag(maybe_flag, midx).into(),
|
||||
ModSeq {
|
||||
metadata_item,
|
||||
modseq,
|
||||
} => is_keep_modseq(metadata_item, modseq, midx).into(),
|
||||
|
||||
// All the stuff we can't evaluate yet
|
||||
Bcc(_) | Cc(_) | From(_) | Header(..) | SentBefore(_) | SentOn(_) | SentSince(_)
|
||||
| Subject(_) | To(_) | Before(_) | On(_) | Since(_) | Larger(_) | Smaller(_)
|
||||
| Text(_) | Body(_) => PartialDecision::Postpone,
|
||||
|
||||
unknown => {
|
||||
tracing::error!("Unknown filter {:?}", unknown);
|
||||
PartialDecision::Discard
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// @TODO we re-eveluate twice the same logic. The correct way would be, on each pass,
|
||||
/// to simplify the searck query, by removing the elements that were already checked.
|
||||
/// For example if we have AND(OR(seqid(X), body(Y)), body(X)), we can't keep for sure
|
||||
/// the email, as body(x) might be false. So we need to check it. But as seqid(x) is true,
|
||||
/// we could simplify the request to just body(x) and truncate the first OR. Today, we are
|
||||
/// not doing that, and thus we reevaluate everything.
|
||||
pub fn is_keep_on_query(&self, mail_view: &MailView) -> bool {
|
||||
use SearchKey::*;
|
||||
match self.0 {
|
||||
// Combinator logic
|
||||
And(expr_list) => expr_list
|
||||
.as_ref()
|
||||
.iter()
|
||||
.all(|cur| Criteria(cur).is_keep_on_query(mail_view)),
|
||||
Or(left, right) => {
|
||||
Criteria(left).is_keep_on_query(mail_view)
|
||||
|| Criteria(right).is_keep_on_query(mail_view)
|
||||
}
|
||||
Not(expr) => !Criteria(expr).is_keep_on_query(mail_view),
|
||||
All => true,
|
||||
|
||||
//@FIXME Reevaluating our previous logic...
|
||||
maybe_seq if is_sk_seq(maybe_seq) => is_keep_seq(maybe_seq, &mail_view.in_idx),
|
||||
maybe_flag if is_sk_flag(maybe_flag) => is_keep_flag(maybe_flag, &mail_view.in_idx),
|
||||
ModSeq {
|
||||
metadata_item,
|
||||
modseq,
|
||||
} => is_keep_modseq(metadata_item, modseq, &mail_view.in_idx).into(),
|
||||
|
||||
// Filter on mail meta
|
||||
Before(search_naive) => match mail_view.stored_naive_date() {
|
||||
Ok(msg_naive) => &msg_naive < search_naive.as_ref(),
|
||||
_ => false,
|
||||
},
|
||||
On(search_naive) => match mail_view.stored_naive_date() {
|
||||
Ok(msg_naive) => &msg_naive == search_naive.as_ref(),
|
||||
_ => false,
|
||||
},
|
||||
Since(search_naive) => match mail_view.stored_naive_date() {
|
||||
Ok(msg_naive) => &msg_naive > search_naive.as_ref(),
|
||||
_ => false,
|
||||
},
|
||||
|
||||
// Message size is also stored in MailMeta
|
||||
Larger(size_ref) => {
|
||||
mail_view
|
||||
.query_result
|
||||
.metadata()
|
||||
.expect("metadata were fetched")
|
||||
.rfc822_size
|
||||
> *size_ref as usize
|
||||
}
|
||||
Smaller(size_ref) => {
|
||||
mail_view
|
||||
.query_result
|
||||
.metadata()
|
||||
.expect("metadata were fetched")
|
||||
.rfc822_size
|
||||
< *size_ref as usize
|
||||
}
|
||||
|
||||
// Filter on well-known headers
|
||||
Bcc(txt) => mail_view.is_header_contains_pattern(&b"bcc"[..], txt.as_ref()),
|
||||
Cc(txt) => mail_view.is_header_contains_pattern(&b"cc"[..], txt.as_ref()),
|
||||
From(txt) => mail_view.is_header_contains_pattern(&b"from"[..], txt.as_ref()),
|
||||
Subject(txt) => mail_view.is_header_contains_pattern(&b"subject"[..], txt.as_ref()),
|
||||
To(txt) => mail_view.is_header_contains_pattern(&b"to"[..], txt.as_ref()),
|
||||
Header(hdr, txt) => mail_view.is_header_contains_pattern(hdr.as_ref(), txt.as_ref()),
|
||||
|
||||
// Filter on Date header
|
||||
SentBefore(search_naive) => mail_view
|
||||
.imf()
|
||||
.map(|imf| imf.naive_date().ok())
|
||||
.flatten()
|
||||
.map(|msg_naive| &msg_naive < search_naive.as_ref())
|
||||
.unwrap_or(false),
|
||||
SentOn(search_naive) => mail_view
|
||||
.imf()
|
||||
.map(|imf| imf.naive_date().ok())
|
||||
.flatten()
|
||||
.map(|msg_naive| &msg_naive == search_naive.as_ref())
|
||||
.unwrap_or(false),
|
||||
SentSince(search_naive) => mail_view
|
||||
.imf()
|
||||
.map(|imf| imf.naive_date().ok())
|
||||
.flatten()
|
||||
.map(|msg_naive| &msg_naive > search_naive.as_ref())
|
||||
.unwrap_or(false),
|
||||
|
||||
// Filter on the full content of the email
|
||||
Text(txt) => mail_view
|
||||
.content
|
||||
.as_msg()
|
||||
.map(|msg| {
|
||||
msg.raw_part
|
||||
.windows(txt.as_ref().len())
|
||||
.any(|win| win == txt.as_ref())
|
||||
})
|
||||
.unwrap_or(false),
|
||||
Body(txt) => mail_view
|
||||
.content
|
||||
.as_msg()
|
||||
.map(|msg| {
|
||||
msg.raw_body
|
||||
.windows(txt.as_ref().len())
|
||||
.any(|win| win == txt.as_ref())
|
||||
})
|
||||
.unwrap_or(false),
|
||||
|
||||
unknown => {
|
||||
tracing::error!("Unknown filter {:?}", unknown);
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Sequence things ----
|
||||
fn sequence_set_all() -> SequenceSet {
|
||||
SequenceSet::from(Sequence::Range(
|
||||
SeqOrUid::Value(NonZeroU32::MIN),
|
||||
SeqOrUid::Asterisk,
|
||||
))
|
||||
}
|
||||
|
||||
// This is wrong as sequences can overlap
|
||||
fn approx_sequence_set_size(seq_set: &SequenceSet) -> u64 {
|
||||
seq_set.0.as_ref().iter().fold(0u64, |acc, seq| {
|
||||
acc.saturating_add(approx_sequence_size(seq))
|
||||
})
|
||||
}
|
||||
|
||||
// This is wrong as sequence UID can have holes,
|
||||
// as we don't know the number of messages in the mailbox also
|
||||
// we gave to guess
|
||||
fn approx_sequence_size(seq: &Sequence) -> u64 {
|
||||
match seq {
|
||||
Sequence::Single(_) => 1,
|
||||
Sequence::Range(SeqOrUid::Asterisk, _) | Sequence::Range(_, SeqOrUid::Asterisk) => u64::MAX,
|
||||
Sequence::Range(SeqOrUid::Value(x1), SeqOrUid::Value(x2)) => {
|
||||
let x2 = x2.get() as i64;
|
||||
let x1 = x1.get() as i64;
|
||||
(x2 - x1).abs().try_into().unwrap_or(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Partial decision things ----
|
||||
|
||||
enum PartialDecision {
|
||||
Keep,
|
||||
Discard,
|
||||
Postpone,
|
||||
}
|
||||
impl From<bool> for PartialDecision {
|
||||
fn from(x: bool) -> Self {
|
||||
match x {
|
||||
true => PartialDecision::Keep,
|
||||
_ => PartialDecision::Discard,
|
||||
}
|
||||
}
|
||||
}
|
||||
impl PartialDecision {
|
||||
fn not(&self) -> Self {
|
||||
match self {
|
||||
Self::Keep => Self::Discard,
|
||||
Self::Discard => Self::Keep,
|
||||
Self::Postpone => Self::Postpone,
|
||||
}
|
||||
}
|
||||
|
||||
fn or(&self, other: &Self) -> Self {
|
||||
match (self, other) {
|
||||
(Self::Keep, _) | (_, Self::Keep) => Self::Keep,
|
||||
(Self::Postpone, _) | (_, Self::Postpone) => Self::Postpone,
|
||||
(Self::Discard, Self::Discard) => Self::Discard,
|
||||
}
|
||||
}
|
||||
|
||||
fn and(&self, other: &Self) -> Self {
|
||||
match (self, other) {
|
||||
(Self::Discard, _) | (_, Self::Discard) => Self::Discard,
|
||||
(Self::Postpone, _) | (_, Self::Postpone) => Self::Postpone,
|
||||
(Self::Keep, Self::Keep) => Self::Keep,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_keep(&self) -> bool {
|
||||
!matches!(self, Self::Discard)
|
||||
}
|
||||
}
|
||||
|
||||
// ----- Search Key things ---
|
||||
fn is_sk_flag(sk: &SearchKey) -> bool {
|
||||
use SearchKey::*;
|
||||
match sk {
|
||||
Answered | Deleted | Draft | Flagged | Keyword(..) | New | Old | Recent | Seen
|
||||
| Unanswered | Undeleted | Undraft | Unflagged | Unkeyword(..) | Unseen => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_keep_flag(sk: &SearchKey, midx: &MailIndex) -> bool {
|
||||
use SearchKey::*;
|
||||
match sk {
|
||||
Answered => midx.is_flag_set("\\Answered"),
|
||||
Deleted => midx.is_flag_set("\\Deleted"),
|
||||
Draft => midx.is_flag_set("\\Draft"),
|
||||
Flagged => midx.is_flag_set("\\Flagged"),
|
||||
Keyword(kw) => midx.is_flag_set(kw.inner()),
|
||||
New => {
|
||||
let is_recent = midx.is_flag_set("\\Recent");
|
||||
let is_seen = midx.is_flag_set("\\Seen");
|
||||
is_recent && !is_seen
|
||||
}
|
||||
Old => {
|
||||
let is_recent = midx.is_flag_set("\\Recent");
|
||||
!is_recent
|
||||
}
|
||||
Recent => midx.is_flag_set("\\Recent"),
|
||||
Seen => midx.is_flag_set("\\Seen"),
|
||||
Unanswered => {
|
||||
let is_answered = midx.is_flag_set("\\Recent");
|
||||
!is_answered
|
||||
}
|
||||
Undeleted => {
|
||||
let is_deleted = midx.is_flag_set("\\Deleted");
|
||||
!is_deleted
|
||||
}
|
||||
Undraft => {
|
||||
let is_draft = midx.is_flag_set("\\Draft");
|
||||
!is_draft
|
||||
}
|
||||
Unflagged => {
|
||||
let is_flagged = midx.is_flag_set("\\Flagged");
|
||||
!is_flagged
|
||||
}
|
||||
Unkeyword(kw) => {
|
||||
let is_keyword_set = midx.is_flag_set(kw.inner());
|
||||
!is_keyword_set
|
||||
}
|
||||
Unseen => {
|
||||
let is_seen = midx.is_flag_set("\\Seen");
|
||||
!is_seen
|
||||
}
|
||||
|
||||
// Not flag logic
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_sk_seq(sk: &SearchKey) -> bool {
|
||||
use SearchKey::*;
|
||||
match sk {
|
||||
SequenceSet(..) | Uid(..) => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
fn is_keep_seq(sk: &SearchKey, midx: &MailIndex) -> bool {
|
||||
use SearchKey::*;
|
||||
match sk {
|
||||
SequenceSet(seq_set) => seq_set
|
||||
.0
|
||||
.as_ref()
|
||||
.iter()
|
||||
.any(|seq| midx.is_in_sequence_i(seq)),
|
||||
Uid(seq_set) => seq_set
|
||||
.0
|
||||
.as_ref()
|
||||
.iter()
|
||||
.any(|seq| midx.is_in_sequence_uid(seq)),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_keep_modseq(
|
||||
filter: &Option<MetadataItemSearch>,
|
||||
modseq: &NonZeroU64,
|
||||
midx: &MailIndex,
|
||||
) -> bool {
|
||||
if filter.is_some() {
|
||||
tracing::warn!(filter=?filter, "Ignoring search metadata filter as it's not supported yet");
|
||||
}
|
||||
modseq <= &midx.modseq
|
||||
}
|
||||
|
|
@ -1,173 +1,159 @@
|
|||
use crate::imap::capability::{ClientCapability, ServerCapability};
|
||||
use anyhow::Error;
|
||||
use boitalettres::errors::Error as BalError;
|
||||
use boitalettres::proto::{Request, Response};
|
||||
use futures::future::BoxFuture;
|
||||
use futures::future::FutureExt;
|
||||
use imap_codec::types::response::{Response as ImapRes, Status};
|
||||
use tokio::sync::mpsc::error::TrySendError;
|
||||
use tokio::sync::{mpsc, oneshot};
|
||||
|
||||
use crate::imap::command::{anonymous, authenticated, selected};
|
||||
use crate::imap::flow;
|
||||
use crate::imap::request::Request;
|
||||
use crate::imap::response::{Response, ResponseOrIdle};
|
||||
use crate::login::ArcLoginProvider;
|
||||
use anyhow::{anyhow, bail, Context, Result};
|
||||
use imap_codec::imap_types::{command::Command, core::Tag};
|
||||
|
||||
/* This constant configures backpressure in the system,
|
||||
* or more specifically, how many pipelined messages are allowed
|
||||
* before refusing them
|
||||
*/
|
||||
const MAX_PIPELINED_COMMANDS: usize = 10;
|
||||
|
||||
struct Message {
|
||||
req: Request,
|
||||
tx: oneshot::Sender<Result<Response, BalError>>,
|
||||
}
|
||||
|
||||
//-----
|
||||
|
||||
pub struct Manager {
|
||||
tx: mpsc::Sender<Message>,
|
||||
}
|
||||
|
||||
impl Manager {
|
||||
pub fn new(login_provider: ArcLoginProvider) -> Self {
|
||||
let (tx, rx) = mpsc::channel(MAX_PIPELINED_COMMANDS);
|
||||
tokio::spawn(async move {
|
||||
let mut instance = Instance::new(login_provider, rx);
|
||||
instance.start().await;
|
||||
});
|
||||
Self { tx }
|
||||
}
|
||||
|
||||
pub fn process(&self, req: Request) -> BoxFuture<'static, Result<Response, BalError>> {
|
||||
let (tx, rx) = oneshot::channel();
|
||||
let msg = Message { req, tx };
|
||||
|
||||
// We use try_send on a bounded channel to protect the daemons from DoS.
|
||||
// Pipelining requests in IMAP are a special case: they should not occure often
|
||||
// and in a limited number (like 3 requests). Someone filling the channel
|
||||
// will probably be malicious so we "rate limit" them.
|
||||
match self.tx.try_send(msg) {
|
||||
Ok(()) => (),
|
||||
Err(TrySendError::Full(_)) => {
|
||||
return async { Response::bad("Too fast! Send less pipelined requests.") }.boxed()
|
||||
}
|
||||
Err(TrySendError::Closed(_)) => {
|
||||
return async { Err(BalError::Text("Terminated session".to_string())) }.boxed()
|
||||
}
|
||||
};
|
||||
|
||||
// @FIXME add a timeout, handle a session that fails.
|
||||
async {
|
||||
match rx.await {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
tracing::warn!("Got error {:#?}", e);
|
||||
Response::bad("No response from the session handler")
|
||||
}
|
||||
}
|
||||
}
|
||||
.boxed()
|
||||
}
|
||||
}
|
||||
|
||||
//-----
|
||||
|
||||
pub struct InnerContext<'a> {
|
||||
pub req: &'a Request,
|
||||
pub state: &'a flow::State,
|
||||
pub login: &'a ArcLoginProvider,
|
||||
}
|
||||
|
||||
pub struct Instance {
|
||||
rx: mpsc::Receiver<Message>,
|
||||
|
||||
pub login_provider: ArcLoginProvider,
|
||||
pub server_capabilities: ServerCapability,
|
||||
pub client_capabilities: ClientCapability,
|
||||
pub state: flow::State,
|
||||
}
|
||||
impl Instance {
|
||||
pub fn new(login_provider: ArcLoginProvider, cap: ServerCapability) -> Self {
|
||||
let client_cap = ClientCapability::new(&cap);
|
||||
fn new(login_provider: ArcLoginProvider, rx: mpsc::Receiver<Message>) -> Self {
|
||||
Self {
|
||||
login_provider,
|
||||
rx,
|
||||
state: flow::State::NotAuthenticated,
|
||||
server_capabilities: cap,
|
||||
client_capabilities: client_cap,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn request(&mut self, req: Request) -> ResponseOrIdle {
|
||||
match req {
|
||||
Request::IdleStart(tag) => self.idle_init(tag),
|
||||
Request::IdlePoll => self.idle_poll().await,
|
||||
Request::ImapCommand(cmd) => self.command(cmd).await,
|
||||
}
|
||||
}
|
||||
//@FIXME add a function that compute the runner's name from its local info
|
||||
// to ease debug
|
||||
// fn name(&self) -> String { }
|
||||
|
||||
pub fn idle_init(&mut self, tag: Tag<'static>) -> ResponseOrIdle {
|
||||
// Build transition
|
||||
//@FIXME the notifier should be hidden inside the state and thus not part of the transition!
|
||||
let transition = flow::Transition::Idle(tag.clone(), tokio::sync::Notify::new());
|
||||
async fn start(mut self) {
|
||||
//@FIXME add more info about the runner
|
||||
tracing::debug!("starting runner");
|
||||
|
||||
// Try to apply the transition and get the stop notifier
|
||||
let maybe_stop = self
|
||||
.state
|
||||
.apply(transition)
|
||||
.context("IDLE transition failed")
|
||||
.and_then(|_| {
|
||||
self.state
|
||||
.notify()
|
||||
.ok_or(anyhow!("IDLE state has no Notify object"))
|
||||
while let Some(msg) = self.rx.recv().await {
|
||||
let ctx = InnerContext {
|
||||
req: &msg.req,
|
||||
state: &self.state,
|
||||
login: &self.login_provider,
|
||||
};
|
||||
|
||||
// Command behavior is modulated by the state.
|
||||
// To prevent state error, we handle the same command in separate code paths.
|
||||
let ctrl = match &self.state {
|
||||
flow::State::NotAuthenticated => anonymous::dispatch(ctx).await,
|
||||
flow::State::Authenticated(user) => authenticated::dispatch(ctx, user).await,
|
||||
flow::State::Selected(user, mailbox) => {
|
||||
selected::dispatch(ctx, user, mailbox).await
|
||||
}
|
||||
_ => Response::bad("No commands are allowed in the LOGOUT state.")
|
||||
.map(|r| (r, flow::Transition::No))
|
||||
.map_err(Error::msg),
|
||||
};
|
||||
|
||||
// Process result
|
||||
let res = match ctrl {
|
||||
Ok((res, tr)) => {
|
||||
//@FIXME remove unwrap
|
||||
self.state = self.state.apply(tr).unwrap();
|
||||
|
||||
//@FIXME enrich here the command with some global status
|
||||
|
||||
Ok(res)
|
||||
}
|
||||
// Cast from anyhow::Error to Bal::Error
|
||||
// @FIXME proper error handling would be great
|
||||
Err(e) => match e.downcast::<BalError>() {
|
||||
Ok(be) => Err(be),
|
||||
Err(e) => {
|
||||
tracing::warn!(error=%e, "internal.error");
|
||||
Response::bad("Internal error")
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
//@FIXME I think we should quit this thread on error and having our manager watch it,
|
||||
// and then abort the session as it is corrupted.
|
||||
msg.tx.send(res).unwrap_or_else(|e| {
|
||||
tracing::warn!("failed to send imap response to manager: {:#?}", e)
|
||||
});
|
||||
|
||||
// Build an appropriate response
|
||||
match maybe_stop {
|
||||
Ok(stop) => ResponseOrIdle::IdleAccept(stop),
|
||||
Err(e) => {
|
||||
tracing::error!(err=?e, "unable to init idle due to a transition error");
|
||||
//ResponseOrIdle::IdleReject(tag)
|
||||
let no = Response::build()
|
||||
.tag(tag)
|
||||
.message(
|
||||
"Internal error, processing command triggered an illegal IMAP state transition",
|
||||
)
|
||||
.no()
|
||||
.unwrap();
|
||||
ResponseOrIdle::IdleReject(no)
|
||||
if let flow::State::Logout = &self.state {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn idle_poll(&mut self) -> ResponseOrIdle {
|
||||
match self.idle_poll_happy().await {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
tracing::error!(err=?e, "something bad happened in idle");
|
||||
ResponseOrIdle::Response(Response::bye().unwrap())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn idle_poll_happy(&mut self) -> Result<ResponseOrIdle> {
|
||||
let (mbx, tag, stop) = match &mut self.state {
|
||||
flow::State::Idle(_, ref mut mbx, _, tag, stop) => (mbx, tag.clone(), stop.clone()),
|
||||
_ => bail!("Invalid session state, can't idle"),
|
||||
};
|
||||
|
||||
tokio::select! {
|
||||
_ = stop.notified() => {
|
||||
self.state.apply(flow::Transition::UnIdle)?;
|
||||
return Ok(ResponseOrIdle::Response(Response::build()
|
||||
.tag(tag.clone())
|
||||
.message("IDLE completed")
|
||||
.ok()?))
|
||||
},
|
||||
change = mbx.idle_sync() => {
|
||||
tracing::debug!("idle event");
|
||||
return Ok(ResponseOrIdle::IdleEvent(change?));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn command(&mut self, cmd: Command<'static>) -> ResponseOrIdle {
|
||||
// Command behavior is modulated by the state.
|
||||
// To prevent state error, we handle the same command in separate code paths.
|
||||
let (resp, tr) = match &mut self.state {
|
||||
flow::State::NotAuthenticated => {
|
||||
let ctx = anonymous::AnonymousContext {
|
||||
req: &cmd,
|
||||
login_provider: &self.login_provider,
|
||||
server_capabilities: &self.server_capabilities,
|
||||
};
|
||||
anonymous::dispatch(ctx).await
|
||||
}
|
||||
flow::State::Authenticated(ref user) => {
|
||||
let ctx = authenticated::AuthenticatedContext {
|
||||
req: &cmd,
|
||||
server_capabilities: &self.server_capabilities,
|
||||
client_capabilities: &mut self.client_capabilities,
|
||||
user,
|
||||
};
|
||||
authenticated::dispatch(ctx).await
|
||||
}
|
||||
flow::State::Selected(ref user, ref mut mailbox, ref perm) => {
|
||||
let ctx = selected::SelectedContext {
|
||||
req: &cmd,
|
||||
server_capabilities: &self.server_capabilities,
|
||||
client_capabilities: &mut self.client_capabilities,
|
||||
user,
|
||||
mailbox,
|
||||
perm,
|
||||
};
|
||||
selected::dispatch(ctx).await
|
||||
}
|
||||
flow::State::Idle(..) => Err(anyhow!("can not receive command while idling")),
|
||||
flow::State::Logout => Response::build()
|
||||
.tag(cmd.tag.clone())
|
||||
.message("No commands are allowed in the LOGOUT state.")
|
||||
.bad()
|
||||
.map(|r| (r, flow::Transition::None)),
|
||||
}
|
||||
.unwrap_or_else(|err| {
|
||||
tracing::error!("Command error {:?} occured while processing {:?}", err, cmd);
|
||||
(
|
||||
Response::build()
|
||||
.to_req(&cmd)
|
||||
.message("Internal error while processing command")
|
||||
.bad()
|
||||
.unwrap(),
|
||||
flow::Transition::None,
|
||||
)
|
||||
});
|
||||
|
||||
if let Err(e) = self.state.apply(tr) {
|
||||
tracing::error!(
|
||||
"Transition error {:?} occured while processing on command {:?}",
|
||||
e,
|
||||
cmd
|
||||
);
|
||||
return ResponseOrIdle::Response(Response::build()
|
||||
.to_req(&cmd)
|
||||
.message(
|
||||
"Internal error, processing command triggered an illegal IMAP state transition",
|
||||
)
|
||||
.bad()
|
||||
.unwrap());
|
||||
}
|
||||
ResponseOrIdle::Response(resp)
|
||||
|
||||
/*match &self.state {
|
||||
flow::State::Idle(_, _, _, _, n) => ResponseOrIdle::StartIdle(n.clone()),
|
||||
_ => ResponseOrIdle::Response(resp),
|
||||
}*/
|
||||
//@FIXME add more info about the runner
|
||||
tracing::debug!("exiting runner");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,26 +0,0 @@
|
|||
/*
|
||||
use anyhow::Result;
|
||||
// ---- UTIL: function to wait for a value to have changed in K2V ----
|
||||
|
||||
pub async fn k2v_wait_value_changed(
|
||||
k2v: &storage::RowStore,
|
||||
key: &storage::RowRef,
|
||||
) -> Result<CausalValue> {
|
||||
loop {
|
||||
if let Some(ct) = prev_ct {
|
||||
match k2v.poll_item(pk, sk, ct.clone(), None).await? {
|
||||
None => continue,
|
||||
Some(cv) => return Ok(cv),
|
||||
}
|
||||
} else {
|
||||
match k2v.read_item(pk, sk).await {
|
||||
Err(k2v_client::Error::NotFound) => {
|
||||
k2v.insert_item(pk, sk, vec![0u8], None).await?;
|
||||
}
|
||||
Err(e) => return Err(e.into()),
|
||||
Ok(cv) => return Ok(cv),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
*/
|
||||
146
src/lmtp.rs
146
src/lmtp.rs
|
|
@ -1,27 +1,26 @@
|
|||
use std::collections::HashMap;
|
||||
use std::net::SocketAddr;
|
||||
use std::{pin::Pin, sync::Arc};
|
||||
|
||||
use anyhow::Result;
|
||||
use anyhow::{bail, Result};
|
||||
use async_trait::async_trait;
|
||||
use duplexify::Duplex;
|
||||
use futures::{io, AsyncRead, AsyncReadExt, AsyncWrite};
|
||||
use futures::{
|
||||
stream,
|
||||
stream::{FuturesOrdered, FuturesUnordered},
|
||||
StreamExt,
|
||||
};
|
||||
use futures::{stream, stream::FuturesUnordered, StreamExt};
|
||||
use log::*;
|
||||
use tokio::net::TcpListener;
|
||||
use rusoto_s3::{PutObjectRequest, S3Client, S3};
|
||||
use tokio::net::{TcpListener, TcpStream};
|
||||
use tokio::select;
|
||||
use tokio::sync::watch;
|
||||
use tokio_util::compat::*;
|
||||
|
||||
use smtp_message::{DataUnescaper, Email, EscapedDataReader, Reply, ReplyCode};
|
||||
use smtp_server::{reply, Config, ConnectionMetadata, Decision, MailMetadata};
|
||||
use smtp_message::{Email, EscapedDataReader, Reply, ReplyCode};
|
||||
use smtp_server::{reply, Config, ConnectionMetadata, Decision, MailMetadata, Protocol};
|
||||
|
||||
use crate::config::*;
|
||||
use crate::cryptoblob::*;
|
||||
use crate::login::*;
|
||||
use crate::mail::incoming::EncryptedMessage;
|
||||
use crate::mail::mail_ident::*;
|
||||
|
||||
pub struct LmtpServer {
|
||||
bind_addr: SocketAddr,
|
||||
|
|
@ -43,8 +42,6 @@ impl LmtpServer {
|
|||
|
||||
pub async fn run(self: &Arc<Self>, mut must_exit: watch::Receiver<bool>) -> Result<()> {
|
||||
let tcp = TcpListener::bind(self.bind_addr).await?;
|
||||
info!("LMTP server listening on {:#}", self.bind_addr);
|
||||
|
||||
let mut connections = FuturesUnordered::new();
|
||||
|
||||
while !*must_exit.borrow() {
|
||||
|
|
@ -60,12 +57,11 @@ impl LmtpServer {
|
|||
_ = wait_conn_finished => continue,
|
||||
_ = must_exit.changed() => continue,
|
||||
};
|
||||
info!("LMTP: accepted connection from {}", remote_addr);
|
||||
|
||||
let conn = tokio::spawn(smtp_server::interact(
|
||||
socket.compat(),
|
||||
smtp_server::IsAlreadyTls::No,
|
||||
(),
|
||||
Conn { remote_addr },
|
||||
self.clone(),
|
||||
));
|
||||
|
||||
|
|
@ -82,6 +78,10 @@ impl LmtpServer {
|
|||
|
||||
// ----
|
||||
|
||||
pub struct Conn {
|
||||
remote_addr: SocketAddr,
|
||||
}
|
||||
|
||||
pub struct Message {
|
||||
to: Vec<PublicCredentials>,
|
||||
}
|
||||
|
|
@ -90,21 +90,21 @@ pub struct Message {
|
|||
impl Config for LmtpServer {
|
||||
type Protocol = smtp_server::protocol::Lmtp;
|
||||
|
||||
type ConnectionUserMeta = ();
|
||||
type ConnectionUserMeta = Conn;
|
||||
type MailUserMeta = Message;
|
||||
|
||||
fn hostname(&self, _conn_meta: &ConnectionMetadata<()>) -> &str {
|
||||
fn hostname(&self, _conn_meta: &ConnectionMetadata<Conn>) -> &str {
|
||||
&self.hostname
|
||||
}
|
||||
|
||||
async fn new_mail(&self, _conn_meta: &mut ConnectionMetadata<()>) -> Message {
|
||||
async fn new_mail(&self, _conn_meta: &mut ConnectionMetadata<Conn>) -> Message {
|
||||
Message { to: vec![] }
|
||||
}
|
||||
|
||||
async fn tls_accept<IO>(
|
||||
&self,
|
||||
_io: IO,
|
||||
_conn_meta: &mut ConnectionMetadata<()>,
|
||||
_conn_meta: &mut ConnectionMetadata<Conn>,
|
||||
) -> io::Result<Duplex<Pin<Box<dyn Send + AsyncRead>>, Pin<Box<dyn Send + AsyncWrite>>>>
|
||||
where
|
||||
IO: Send + AsyncRead + AsyncWrite,
|
||||
|
|
@ -118,8 +118,8 @@ impl Config for LmtpServer {
|
|||
async fn filter_from(
|
||||
&self,
|
||||
from: Option<Email>,
|
||||
_meta: &mut MailMetadata<Message>,
|
||||
_conn_meta: &mut ConnectionMetadata<()>,
|
||||
meta: &mut MailMetadata<Message>,
|
||||
_conn_meta: &mut ConnectionMetadata<Conn>,
|
||||
) -> Decision<Option<Email>> {
|
||||
Decision::Accept {
|
||||
reply: reply::okay_from().convert(),
|
||||
|
|
@ -131,7 +131,7 @@ impl Config for LmtpServer {
|
|||
&self,
|
||||
to: Email,
|
||||
meta: &mut MailMetadata<Message>,
|
||||
_conn_meta: &mut ConnectionMetadata<()>,
|
||||
_conn_meta: &mut ConnectionMetadata<Conn>,
|
||||
) -> Decision<Email> {
|
||||
let to_str = match to.hostname.as_ref() {
|
||||
Some(h) => format!("{}@{}", to.localpart, h),
|
||||
|
|
@ -155,14 +155,17 @@ impl Config for LmtpServer {
|
|||
}
|
||||
}
|
||||
|
||||
async fn handle_mail<'resp, R>(
|
||||
&'resp self,
|
||||
reader: &mut EscapedDataReader<'_, R>,
|
||||
async fn handle_mail<'a, 'slife0, 'slife1, 'stream, R>(
|
||||
&'slife0 self,
|
||||
reader: &mut EscapedDataReader<'a, R>,
|
||||
meta: MailMetadata<Message>,
|
||||
_conn_meta: &'resp mut ConnectionMetadata<()>,
|
||||
) -> Pin<Box<dyn futures::Stream<Item = Decision<()>> + Send + 'resp>>
|
||||
conn_meta: &'slife1 mut ConnectionMetadata<Conn>,
|
||||
) -> Pin<Box<dyn futures::Stream<Item = Decision<()>> + Send + 'stream>>
|
||||
where
|
||||
R: Send + Unpin + AsyncRead,
|
||||
'slife0: 'stream,
|
||||
'slife1: 'stream,
|
||||
Self: 'stream,
|
||||
{
|
||||
let err_response_stream = |meta: MailMetadata<Message>, msg: String| {
|
||||
Box::pin(
|
||||
|
|
@ -177,45 +180,72 @@ impl Config for LmtpServer {
|
|||
};
|
||||
|
||||
let mut text = Vec::new();
|
||||
if let Err(e) = reader.read_to_end(&mut text).await {
|
||||
return err_response_stream(meta, format!("io error: {}", e));
|
||||
if reader.read_to_end(&mut text).await.is_err() {
|
||||
return err_response_stream(meta, "io error".into());
|
||||
}
|
||||
reader.complete();
|
||||
let raw_size = text.len();
|
||||
|
||||
// Unescape email, shrink it also to remove last dot
|
||||
let unesc_res = DataUnescaper::new(true).unescape(&mut text);
|
||||
text.truncate(unesc_res.written);
|
||||
tracing::debug!(prev_sz = raw_size, new_sz = text.len(), "unescaped");
|
||||
|
||||
let encrypted_message = match EncryptedMessage::new(text) {
|
||||
Ok(x) => Arc::new(x),
|
||||
Err(e) => return err_response_stream(meta, e.to_string()),
|
||||
};
|
||||
|
||||
Box::pin(
|
||||
meta.user
|
||||
.to
|
||||
.into_iter()
|
||||
.map(move |creds| {
|
||||
let encrypted_message = encrypted_message.clone();
|
||||
async move {
|
||||
match encrypted_message.deliver_to(creds).await {
|
||||
Ok(()) => Decision::Accept {
|
||||
reply: reply::okay_mail().convert(),
|
||||
res: (),
|
||||
},
|
||||
Err(e) => Decision::Reject {
|
||||
reply: Reply {
|
||||
code: ReplyCode::POLICY_REASON,
|
||||
ecode: None,
|
||||
text: vec![smtp_message::MaybeUtf8::Utf8(e.to_string())],
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
})
|
||||
.collect::<FuturesOrdered<_>>(),
|
||||
)
|
||||
Box::pin(stream::iter(meta.user.to.into_iter()).then(move |creds| {
|
||||
let encrypted_message = encrypted_message.clone();
|
||||
async move {
|
||||
match encrypted_message.deliver_to(creds).await {
|
||||
Ok(()) => Decision::Accept {
|
||||
reply: reply::okay_mail().convert(),
|
||||
res: (),
|
||||
},
|
||||
Err(e) => Decision::Reject {
|
||||
reply: Reply {
|
||||
code: ReplyCode::POLICY_REASON,
|
||||
ecode: None,
|
||||
text: vec![smtp_message::MaybeUtf8::Utf8(e.to_string())],
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
// ----
|
||||
|
||||
struct EncryptedMessage {
|
||||
key: Key,
|
||||
encrypted_body: Vec<u8>,
|
||||
}
|
||||
|
||||
impl EncryptedMessage {
|
||||
fn new(body: Vec<u8>) -> Result<Self> {
|
||||
let key = gen_key();
|
||||
let encrypted_body = seal(&body, &key)?;
|
||||
Ok(Self {
|
||||
key,
|
||||
encrypted_body,
|
||||
})
|
||||
}
|
||||
|
||||
async fn deliver_to(self: Arc<Self>, creds: PublicCredentials) -> Result<()> {
|
||||
let s3_client = creds.storage.s3_client()?;
|
||||
|
||||
let encrypted_key =
|
||||
sodiumoxide::crypto::sealedbox::seal(self.key.as_ref(), &creds.public_key);
|
||||
let key_header = base64::encode(&encrypted_key);
|
||||
|
||||
let mut por = PutObjectRequest::default();
|
||||
por.bucket = creds.storage.bucket.clone();
|
||||
por.key = format!("incoming/{}", gen_ident().to_string());
|
||||
por.metadata = Some(
|
||||
[("Message-Key".to_string(), key_header)]
|
||||
.into_iter()
|
||||
.collect::<HashMap<_, _>>(),
|
||||
);
|
||||
por.body = Some(self.encrypted_body.clone().into());
|
||||
s3_client.put_object(por).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,51 +0,0 @@
|
|||
use crate::login::*;
|
||||
use crate::storage::*;
|
||||
|
||||
pub struct DemoLoginProvider {
|
||||
keys: CryptoKeys,
|
||||
in_memory_store: in_memory::MemDb,
|
||||
}
|
||||
|
||||
impl DemoLoginProvider {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
keys: CryptoKeys::init(),
|
||||
in_memory_store: in_memory::MemDb::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl LoginProvider for DemoLoginProvider {
|
||||
async fn login(&self, username: &str, password: &str) -> Result<Credentials> {
|
||||
tracing::debug!(user=%username, "login");
|
||||
|
||||
if username != "alice" {
|
||||
bail!("user does not exist");
|
||||
}
|
||||
|
||||
if password != "hunter2" {
|
||||
bail!("wrong password");
|
||||
}
|
||||
|
||||
let storage = self.in_memory_store.builder("alice").await;
|
||||
let keys = self.keys.clone();
|
||||
|
||||
Ok(Credentials { storage, keys })
|
||||
}
|
||||
|
||||
async fn public_login(&self, email: &str) -> Result<PublicCredentials> {
|
||||
tracing::debug!(user=%email, "public_login");
|
||||
if email != "alice@example.tld" {
|
||||
bail!("invalid email address");
|
||||
}
|
||||
|
||||
let storage = self.in_memory_store.builder("alice").await;
|
||||
let public_key = self.keys.public.clone();
|
||||
|
||||
Ok(PublicCredentials {
|
||||
storage,
|
||||
public_key,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -2,12 +2,14 @@ use anyhow::Result;
|
|||
use async_trait::async_trait;
|
||||
use ldap3::{LdapConnAsync, Scope, SearchEntry};
|
||||
use log::debug;
|
||||
use rusoto_signature::Region;
|
||||
|
||||
use crate::config::*;
|
||||
use crate::login::*;
|
||||
use crate::storage;
|
||||
|
||||
pub struct LdapLoginProvider {
|
||||
k2v_region: Region,
|
||||
s3_region: Region,
|
||||
ldap_server: String,
|
||||
|
||||
pre_bind_on_login: bool,
|
||||
|
|
@ -17,11 +19,13 @@ pub struct LdapLoginProvider {
|
|||
attrs_to_retrieve: Vec<String>,
|
||||
username_attr: String,
|
||||
mail_attr: String,
|
||||
crypto_root_attr: String,
|
||||
|
||||
storage_specific: StorageSpecific,
|
||||
in_memory_store: storage::in_memory::MemDb,
|
||||
garage_store: storage::garage::GarageRoot,
|
||||
aws_access_key_id_attr: String,
|
||||
aws_secret_access_key_attr: String,
|
||||
user_secret_attr: String,
|
||||
alternate_user_secrets_attr: Option<String>,
|
||||
|
||||
bucket_source: BucketSource,
|
||||
}
|
||||
|
||||
enum BucketSource {
|
||||
|
|
@ -29,16 +33,8 @@ enum BucketSource {
|
|||
Attr(String),
|
||||
}
|
||||
|
||||
enum StorageSpecific {
|
||||
InMemory,
|
||||
Garage {
|
||||
from_config: LdapGarageConfig,
|
||||
bucket_source: BucketSource,
|
||||
},
|
||||
}
|
||||
|
||||
impl LdapLoginProvider {
|
||||
pub fn new(config: LoginLdapConfig) -> Result<Self> {
|
||||
pub fn new(config: LoginLdapConfig, k2v_region: Region, s3_region: Region) -> Result<Self> {
|
||||
let bind_dn_and_pw = match (config.bind_dn, config.bind_password) {
|
||||
(Some(dn), Some(pw)) => Some((dn, pw)),
|
||||
(None, None) => None,
|
||||
|
|
@ -47,6 +43,12 @@ impl LdapLoginProvider {
|
|||
),
|
||||
};
|
||||
|
||||
let bucket_source = match (config.bucket, config.bucket_attr) {
|
||||
(Some(b), None) => BucketSource::Constant(b),
|
||||
(None, Some(a)) => BucketSource::Attr(a),
|
||||
_ => bail!("Must set `bucket` or `bucket_attr`, but not both"),
|
||||
};
|
||||
|
||||
if config.pre_bind_on_login && bind_dn_and_pw.is_none() {
|
||||
bail!("Cannot use `pre_bind_on_login` without setting `bind_dn` and `bind_password`");
|
||||
}
|
||||
|
|
@ -54,35 +56,20 @@ impl LdapLoginProvider {
|
|||
let mut attrs_to_retrieve = vec![
|
||||
config.username_attr.clone(),
|
||||
config.mail_attr.clone(),
|
||||
config.crypto_root_attr.clone(),
|
||||
config.aws_access_key_id_attr.clone(),
|
||||
config.aws_secret_access_key_attr.clone(),
|
||||
config.user_secret_attr.clone(),
|
||||
];
|
||||
|
||||
// storage specific
|
||||
let specific = match config.storage {
|
||||
LdapStorage::InMemory => StorageSpecific::InMemory,
|
||||
LdapStorage::Garage(grgconf) => {
|
||||
attrs_to_retrieve.push(grgconf.aws_access_key_id_attr.clone());
|
||||
attrs_to_retrieve.push(grgconf.aws_secret_access_key_attr.clone());
|
||||
|
||||
let bucket_source =
|
||||
match (grgconf.default_bucket.clone(), grgconf.bucket_attr.clone()) {
|
||||
(Some(b), None) => BucketSource::Constant(b),
|
||||
(None, Some(a)) => BucketSource::Attr(a),
|
||||
_ => bail!("Must set `bucket` or `bucket_attr`, but not both"),
|
||||
};
|
||||
|
||||
if let BucketSource::Attr(a) = &bucket_source {
|
||||
attrs_to_retrieve.push(a.clone());
|
||||
}
|
||||
|
||||
StorageSpecific::Garage {
|
||||
from_config: grgconf,
|
||||
bucket_source,
|
||||
}
|
||||
}
|
||||
};
|
||||
if let Some(a) = &config.alternate_user_secrets_attr {
|
||||
attrs_to_retrieve.push(a.clone());
|
||||
}
|
||||
if let BucketSource::Attr(a) = &bucket_source {
|
||||
attrs_to_retrieve.push(a.clone());
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
k2v_region,
|
||||
s3_region,
|
||||
ldap_server: config.ldap_server,
|
||||
pre_bind_on_login: config.pre_bind_on_login,
|
||||
bind_dn_and_pw,
|
||||
|
|
@ -90,47 +77,29 @@ impl LdapLoginProvider {
|
|||
attrs_to_retrieve,
|
||||
username_attr: config.username_attr,
|
||||
mail_attr: config.mail_attr,
|
||||
crypto_root_attr: config.crypto_root_attr,
|
||||
storage_specific: specific,
|
||||
//@FIXME should be created outside of the login provider
|
||||
//Login provider should return only a cryptoroot + a storage URI
|
||||
//storage URI that should be resolved outside...
|
||||
in_memory_store: storage::in_memory::MemDb::new(),
|
||||
garage_store: storage::garage::GarageRoot::new()?,
|
||||
aws_access_key_id_attr: config.aws_access_key_id_attr,
|
||||
aws_secret_access_key_attr: config.aws_secret_access_key_attr,
|
||||
user_secret_attr: config.user_secret_attr,
|
||||
alternate_user_secrets_attr: config.alternate_user_secrets_attr,
|
||||
bucket_source,
|
||||
})
|
||||
}
|
||||
|
||||
async fn storage_creds_from_ldap_user(&self, user: &SearchEntry) -> Result<Builder> {
|
||||
let storage: Builder = match &self.storage_specific {
|
||||
StorageSpecific::InMemory => {
|
||||
self.in_memory_store
|
||||
.builder(&get_attr(user, &self.username_attr)?)
|
||||
.await
|
||||
}
|
||||
StorageSpecific::Garage {
|
||||
from_config,
|
||||
bucket_source,
|
||||
} => {
|
||||
let aws_access_key_id = get_attr(user, &from_config.aws_access_key_id_attr)?;
|
||||
let aws_secret_access_key =
|
||||
get_attr(user, &from_config.aws_secret_access_key_attr)?;
|
||||
let bucket = match bucket_source {
|
||||
BucketSource::Constant(b) => b.clone(),
|
||||
BucketSource::Attr(a) => get_attr(user, &a)?,
|
||||
};
|
||||
|
||||
self.garage_store.user(storage::garage::GarageConf {
|
||||
region: from_config.aws_region.clone(),
|
||||
s3_endpoint: from_config.s3_endpoint.clone(),
|
||||
k2v_endpoint: from_config.k2v_endpoint.clone(),
|
||||
aws_access_key_id,
|
||||
aws_secret_access_key,
|
||||
bucket,
|
||||
})?
|
||||
}
|
||||
fn storage_creds_from_ldap_user(&self, user: &SearchEntry) -> Result<StorageCredentials> {
|
||||
let aws_access_key_id = get_attr(user, &self.aws_access_key_id_attr)?;
|
||||
let aws_secret_access_key = get_attr(user, &self.aws_secret_access_key_attr)?;
|
||||
let bucket = match &self.bucket_source {
|
||||
BucketSource::Constant(b) => b.clone(),
|
||||
BucketSource::Attr(a) => get_attr(user, a)?,
|
||||
};
|
||||
|
||||
Ok(storage)
|
||||
Ok(StorageCredentials {
|
||||
k2v_region: self.k2v_region.clone(),
|
||||
s3_region: self.s3_region.clone(),
|
||||
aws_access_key_id,
|
||||
aws_secret_access_key,
|
||||
bucket,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -180,16 +149,22 @@ impl LoginProvider for LdapLoginProvider {
|
|||
.context("Invalid password")?;
|
||||
debug!("Ldap login with user name {} successfull", username);
|
||||
|
||||
// cryptography
|
||||
let crstr = get_attr(&user, &self.crypto_root_attr)?;
|
||||
let cr = CryptoRoot(crstr);
|
||||
let keys = cr.crypto_keys(password)?;
|
||||
let storage = self.storage_creds_from_ldap_user(&user)?;
|
||||
|
||||
// storage
|
||||
let storage = self.storage_creds_from_ldap_user(&user).await?;
|
||||
let user_secret = get_attr(&user, &self.user_secret_attr)?;
|
||||
let alternate_user_secrets = match &self.alternate_user_secrets_attr {
|
||||
None => vec![],
|
||||
Some(a) => user.attrs.get(a).cloned().unwrap_or_default(),
|
||||
};
|
||||
let user_secrets = UserSecrets {
|
||||
user_secret,
|
||||
alternate_user_secrets,
|
||||
};
|
||||
|
||||
drop(ldap);
|
||||
|
||||
let keys = CryptoKeys::open(&storage, &user_secrets, password).await?;
|
||||
|
||||
Ok(Credentials { storage, keys })
|
||||
}
|
||||
|
||||
|
|
@ -227,15 +202,12 @@ impl LoginProvider for LdapLoginProvider {
|
|||
let user = SearchEntry::construct(matches.into_iter().next().unwrap());
|
||||
debug!("Found matching LDAP user for email {}: {}", email, user.dn);
|
||||
|
||||
// cryptography
|
||||
let crstr = get_attr(&user, &self.crypto_root_attr)?;
|
||||
let cr = CryptoRoot(crstr);
|
||||
let public_key = cr.public_key()?;
|
||||
|
||||
// storage
|
||||
let storage = self.storage_creds_from_ldap_user(&user).await?;
|
||||
let storage = self.storage_creds_from_ldap_user(&user)?;
|
||||
drop(ldap);
|
||||
|
||||
let k2v_client = storage.k2v_client()?;
|
||||
let (_, public_key) = CryptoKeys::load_salt_and_public(&k2v_client).await?;
|
||||
|
||||
Ok(PublicCredentials {
|
||||
storage,
|
||||
public_key,
|
||||
|
|
|
|||
618
src/login/mod.rs
618
src/login/mod.rs
|
|
@ -1,16 +1,21 @@
|
|||
pub mod demo_provider;
|
||||
pub mod ldap_provider;
|
||||
pub mod static_provider;
|
||||
|
||||
use base64::Engine;
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::{anyhow, bail, Context, Result};
|
||||
use async_trait::async_trait;
|
||||
use k2v_client::{
|
||||
BatchInsertOp, BatchReadOp, CausalValue, CausalityToken, Filter, K2vClient, K2vValue,
|
||||
};
|
||||
use rand::prelude::*;
|
||||
use rusoto_core::HttpClient;
|
||||
use rusoto_credential::{AwsCredentials, StaticProvider};
|
||||
use rusoto_s3::S3Client;
|
||||
use rusoto_signature::Region;
|
||||
|
||||
use crate::cryptoblob::*;
|
||||
use crate::storage::*;
|
||||
|
||||
/// The trait LoginProvider defines the interface for a login provider that allows
|
||||
/// to retrieve storage and cryptographic credentials for access to a user account
|
||||
|
|
@ -34,7 +39,7 @@ pub type ArcLoginProvider = Arc<dyn LoginProvider + Send + Sync>;
|
|||
#[derive(Clone, Debug)]
|
||||
pub struct Credentials {
|
||||
/// The storage credentials are used to authenticate access to the underlying storage (S3, K2V)
|
||||
pub storage: Builder,
|
||||
pub storage: StorageCredentials,
|
||||
/// The cryptographic keys are used to encrypt and decrypt data stored in S3 and K2V
|
||||
pub keys: CryptoKeys,
|
||||
}
|
||||
|
|
@ -42,93 +47,32 @@ pub struct Credentials {
|
|||
#[derive(Clone, Debug)]
|
||||
pub struct PublicCredentials {
|
||||
/// The storage credentials are used to authenticate access to the underlying storage (S3, K2V)
|
||||
pub storage: Builder,
|
||||
pub storage: StorageCredentials,
|
||||
pub public_key: PublicKey,
|
||||
}
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
pub struct CryptoRoot(pub String);
|
||||
/// The struct StorageCredentials contains access key to an S3 and K2V bucket
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct StorageCredentials {
|
||||
pub s3_region: Region,
|
||||
pub k2v_region: Region,
|
||||
|
||||
impl CryptoRoot {
|
||||
pub fn create_pass(password: &str, k: &CryptoKeys) -> Result<Self> {
|
||||
let bytes = k.password_seal(password)?;
|
||||
let b64 = base64::engine::general_purpose::STANDARD_NO_PAD.encode(bytes);
|
||||
let cr = format!("aero:cryptoroot:pass:{}", b64);
|
||||
Ok(Self(cr))
|
||||
}
|
||||
pub aws_access_key_id: String,
|
||||
pub aws_secret_access_key: String,
|
||||
pub bucket: String,
|
||||
}
|
||||
|
||||
pub fn create_cleartext(k: &CryptoKeys) -> Self {
|
||||
let bytes = k.serialize();
|
||||
let b64 = base64::engine::general_purpose::STANDARD_NO_PAD.encode(bytes);
|
||||
let cr = format!("aero:cryptoroot:cleartext:{}", b64);
|
||||
Self(cr)
|
||||
}
|
||||
|
||||
pub fn create_incoming(pk: &PublicKey) -> Self {
|
||||
let bytes: &[u8] = &pk[..];
|
||||
let b64 = base64::engine::general_purpose::STANDARD_NO_PAD.encode(bytes);
|
||||
let cr = format!("aero:cryptoroot:incoming:{}", b64);
|
||||
Self(cr)
|
||||
}
|
||||
|
||||
pub fn public_key(&self) -> Result<PublicKey> {
|
||||
match self.0.splitn(4, ':').collect::<Vec<&str>>()[..] {
|
||||
["aero", "cryptoroot", "pass", b64blob] => {
|
||||
let blob = base64::engine::general_purpose::STANDARD_NO_PAD.decode(b64blob)?;
|
||||
if blob.len() < 32 {
|
||||
bail!(
|
||||
"Decoded data is {} bytes long, expect at least 32 bytes",
|
||||
blob.len()
|
||||
);
|
||||
}
|
||||
PublicKey::from_slice(&blob[..32]).context("must be a valid public key")
|
||||
}
|
||||
["aero", "cryptoroot", "cleartext", b64blob] => {
|
||||
let blob = base64::engine::general_purpose::STANDARD_NO_PAD.decode(b64blob)?;
|
||||
Ok(CryptoKeys::deserialize(&blob)?.public)
|
||||
}
|
||||
["aero", "cryptoroot", "incoming", b64blob] => {
|
||||
let blob = base64::engine::general_purpose::STANDARD_NO_PAD.decode(b64blob)?;
|
||||
if blob.len() < 32 {
|
||||
bail!(
|
||||
"Decoded data is {} bytes long, expect at least 32 bytes",
|
||||
blob.len()
|
||||
);
|
||||
}
|
||||
PublicKey::from_slice(&blob[..32]).context("must be a valid public key")
|
||||
}
|
||||
["aero", "cryptoroot", "keyring", _] => {
|
||||
bail!("keyring is not yet implemented!")
|
||||
}
|
||||
_ => bail!(format!(
|
||||
"passed string '{}' is not a valid cryptoroot",
|
||||
self.0
|
||||
)),
|
||||
}
|
||||
}
|
||||
pub fn crypto_keys(&self, password: &str) -> Result<CryptoKeys> {
|
||||
match self.0.splitn(4, ':').collect::<Vec<&str>>()[..] {
|
||||
["aero", "cryptoroot", "pass", b64blob] => {
|
||||
let blob = base64::engine::general_purpose::STANDARD_NO_PAD.decode(b64blob)?;
|
||||
CryptoKeys::password_open(password, &blob)
|
||||
}
|
||||
["aero", "cryptoroot", "cleartext", b64blob] => {
|
||||
let blob = base64::engine::general_purpose::STANDARD_NO_PAD.decode(b64blob)?;
|
||||
CryptoKeys::deserialize(&blob)
|
||||
}
|
||||
["aero", "cryptoroot", "incoming", _] => {
|
||||
bail!("incoming cryptoroot does not contain a crypto key!")
|
||||
}
|
||||
["aero", "cryptoroot", "keyring", _] => {
|
||||
bail!("keyring is not yet implemented!")
|
||||
}
|
||||
_ => bail!(format!(
|
||||
"passed string '{}' is not a valid cryptoroot",
|
||||
self.0
|
||||
)),
|
||||
}
|
||||
}
|
||||
/// The struct UserSecrets represents intermediary secrets that are mixed in with the user's
|
||||
/// password when decrypting the cryptographic keys that are stored in their bucket.
|
||||
/// These secrets should be stored somewhere else (e.g. in the LDAP server or in the
|
||||
/// local config file), as an additionnal authentification factor so that the password
|
||||
/// isn't enough just alone to decrypt the content of a user's bucket.
|
||||
pub struct UserSecrets {
|
||||
/// The main user secret that will be used to encrypt keys when a new password is added
|
||||
pub user_secret: String,
|
||||
/// Alternative user secrets that will be tried when decrypting keys that were encrypted
|
||||
/// with old passwords
|
||||
pub alternate_user_secrets: Vec<String>,
|
||||
}
|
||||
|
||||
/// The struct CryptoKeys contains the cryptographic keys used to encrypt and decrypt
|
||||
|
|
@ -145,20 +89,398 @@ pub struct CryptoKeys {
|
|||
|
||||
// ----
|
||||
|
||||
impl Credentials {
|
||||
pub fn k2v_client(&self) -> Result<K2vClient> {
|
||||
self.storage.k2v_client()
|
||||
}
|
||||
pub fn s3_client(&self) -> Result<S3Client> {
|
||||
self.storage.s3_client()
|
||||
}
|
||||
pub fn bucket(&self) -> &str {
|
||||
self.storage.bucket.as_str()
|
||||
}
|
||||
}
|
||||
|
||||
impl StorageCredentials {
|
||||
pub fn k2v_client(&self) -> Result<K2vClient> {
|
||||
let aws_creds = AwsCredentials::new(
|
||||
self.aws_access_key_id.clone(),
|
||||
self.aws_secret_access_key.clone(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
|
||||
Ok(K2vClient::new(
|
||||
self.k2v_region.clone(),
|
||||
self.bucket.clone(),
|
||||
aws_creds,
|
||||
None,
|
||||
)?)
|
||||
}
|
||||
|
||||
pub fn s3_client(&self) -> Result<S3Client> {
|
||||
let aws_creds_provider = StaticProvider::new_minimal(
|
||||
self.aws_access_key_id.clone(),
|
||||
self.aws_secret_access_key.clone(),
|
||||
);
|
||||
|
||||
Ok(S3Client::new_with(
|
||||
HttpClient::new()?,
|
||||
aws_creds_provider,
|
||||
self.s3_region.clone(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
impl CryptoKeys {
|
||||
/// Initialize a new cryptography root
|
||||
pub fn init() -> Self {
|
||||
pub async fn init(
|
||||
storage: &StorageCredentials,
|
||||
user_secrets: &UserSecrets,
|
||||
password: &str,
|
||||
) -> Result<Self> {
|
||||
// Check that salt and public don't exist already
|
||||
let k2v = storage.k2v_client()?;
|
||||
let (salt_ct, public_ct) = Self::check_uninitialized(&k2v).await?;
|
||||
|
||||
// Generate salt for password identifiers
|
||||
let mut ident_salt = [0u8; 32];
|
||||
thread_rng().fill(&mut ident_salt);
|
||||
|
||||
// Generate (public, private) key pair and master key
|
||||
let (public, secret) = gen_keypair();
|
||||
let master = gen_key();
|
||||
CryptoKeys {
|
||||
let keys = CryptoKeys {
|
||||
master,
|
||||
secret,
|
||||
public,
|
||||
}
|
||||
};
|
||||
|
||||
// Generate short password digest (= password identity)
|
||||
let ident = argon2_kdf(&ident_salt, password.as_bytes(), 16)?;
|
||||
|
||||
// Generate salt for KDF
|
||||
let mut kdf_salt = [0u8; 32];
|
||||
thread_rng().fill(&mut kdf_salt);
|
||||
|
||||
// Calculate key for password secret box
|
||||
let password_key = user_secrets.derive_password_key(&kdf_salt, password)?;
|
||||
|
||||
// Seal a secret box that contains our crypto keys
|
||||
let password_sealed = seal(&keys.serialize(), &password_key)?;
|
||||
|
||||
let password_sortkey = format!("password:{}", hex::encode(&ident));
|
||||
let password_blob = [&kdf_salt[..], &password_sealed].concat();
|
||||
|
||||
// Write values to storage
|
||||
k2v.insert_batch(&[
|
||||
k2v_insert_single_key("keys", "salt", salt_ct, &ident_salt),
|
||||
k2v_insert_single_key("keys", "public", public_ct, &keys.public),
|
||||
k2v_insert_single_key("keys", &password_sortkey, None, &password_blob),
|
||||
])
|
||||
.await
|
||||
.context("InsertBatch for salt, public, and password")?;
|
||||
|
||||
Ok(keys)
|
||||
}
|
||||
|
||||
pub async fn init_without_password(
|
||||
storage: &StorageCredentials,
|
||||
master: &Key,
|
||||
secret: &SecretKey,
|
||||
) -> Result<Self> {
|
||||
// Check that salt and public don't exist already
|
||||
let k2v = storage.k2v_client()?;
|
||||
let (salt_ct, public_ct) = Self::check_uninitialized(&k2v).await?;
|
||||
|
||||
// Generate salt for password identifiers
|
||||
let mut ident_salt = [0u8; 32];
|
||||
thread_rng().fill(&mut ident_salt);
|
||||
|
||||
// Create CryptoKeys struct from given keys
|
||||
let public = secret.public_key();
|
||||
let keys = CryptoKeys {
|
||||
master: master.clone(),
|
||||
secret: secret.clone(),
|
||||
public,
|
||||
};
|
||||
|
||||
// Write values to storage
|
||||
k2v.insert_batch(&[
|
||||
k2v_insert_single_key("keys", "salt", salt_ct, &ident_salt),
|
||||
k2v_insert_single_key("keys", "public", public_ct, &keys.public),
|
||||
])
|
||||
.await
|
||||
.context("InsertBatch for salt and public")?;
|
||||
|
||||
Ok(keys)
|
||||
}
|
||||
|
||||
pub async fn open(
|
||||
storage: &StorageCredentials,
|
||||
user_secrets: &UserSecrets,
|
||||
password: &str,
|
||||
) -> Result<Self> {
|
||||
let k2v = storage.k2v_client()?;
|
||||
let (ident_salt, expected_public) = Self::load_salt_and_public(&k2v).await?;
|
||||
|
||||
// Generate short password digest (= password identity)
|
||||
let ident = argon2_kdf(&ident_salt, password.as_bytes(), 16)?;
|
||||
|
||||
// Lookup password blob
|
||||
let password_sortkey = format!("password:{}", hex::encode(&ident));
|
||||
|
||||
let password_blob = {
|
||||
let mut val = match k2v.read_item("keys", &password_sortkey).await {
|
||||
Err(k2v_client::Error::NotFound) => {
|
||||
bail!("invalid password")
|
||||
}
|
||||
x => x?,
|
||||
};
|
||||
if val.value.len() != 1 {
|
||||
bail!("multiple values for password in storage");
|
||||
}
|
||||
match val.value.pop().unwrap() {
|
||||
K2vValue::Value(v) => v,
|
||||
K2vValue::Tombstone => bail!("invalid password"),
|
||||
}
|
||||
};
|
||||
|
||||
// Try to open blob
|
||||
let kdf_salt = &password_blob[..32];
|
||||
let password_openned =
|
||||
user_secrets.try_open_encrypted_keys(&kdf_salt, password, &password_blob[32..])?;
|
||||
|
||||
let keys = Self::deserialize(&password_openned)?;
|
||||
if keys.public != expected_public {
|
||||
bail!("Password public key doesn't match stored public key");
|
||||
}
|
||||
|
||||
Ok(keys)
|
||||
}
|
||||
|
||||
pub async fn open_without_password(
|
||||
storage: &StorageCredentials,
|
||||
master: &Key,
|
||||
secret: &SecretKey,
|
||||
) -> Result<Self> {
|
||||
let k2v = storage.k2v_client()?;
|
||||
let (_ident_salt, expected_public) = Self::load_salt_and_public(&k2v).await?;
|
||||
|
||||
// Create CryptoKeys struct from given keys
|
||||
let public = secret.public_key();
|
||||
let keys = CryptoKeys {
|
||||
master: master.clone(),
|
||||
secret: secret.clone(),
|
||||
public,
|
||||
};
|
||||
|
||||
// Check public key matches
|
||||
if keys.public != expected_public {
|
||||
bail!("Given public key doesn't match stored public key");
|
||||
}
|
||||
|
||||
Ok(keys)
|
||||
}
|
||||
|
||||
pub async fn add_password(
|
||||
&self,
|
||||
storage: &StorageCredentials,
|
||||
user_secrets: &UserSecrets,
|
||||
password: &str,
|
||||
) -> Result<()> {
|
||||
let k2v = storage.k2v_client()?;
|
||||
let (ident_salt, _public) = Self::load_salt_and_public(&k2v).await?;
|
||||
|
||||
// Generate short password digest (= password identity)
|
||||
let ident = argon2_kdf(&ident_salt, password.as_bytes(), 16)?;
|
||||
|
||||
// Generate salt for KDF
|
||||
let mut kdf_salt = [0u8; 32];
|
||||
thread_rng().fill(&mut kdf_salt);
|
||||
|
||||
// Calculate key for password secret box
|
||||
let password_key = user_secrets.derive_password_key(&kdf_salt, password)?;
|
||||
|
||||
// Seal a secret box that contains our crypto keys
|
||||
let password_sealed = seal(&self.serialize(), &password_key)?;
|
||||
|
||||
let password_sortkey = format!("password:{}", hex::encode(&ident));
|
||||
let password_blob = [&kdf_salt[..], &password_sealed].concat();
|
||||
|
||||
// List existing passwords to overwrite existing entry if necessary
|
||||
let ct = match k2v.read_item("keys", &password_sortkey).await {
|
||||
Err(k2v_client::Error::NotFound) => None,
|
||||
v => {
|
||||
let entry = v?;
|
||||
if entry.value.iter().any(|x| matches!(x, K2vValue::Value(_))) {
|
||||
bail!("password already exists");
|
||||
}
|
||||
Some(entry.causality.clone())
|
||||
}
|
||||
};
|
||||
|
||||
// Write values to storage
|
||||
k2v.insert_batch(&[k2v_insert_single_key(
|
||||
"keys",
|
||||
&password_sortkey,
|
||||
ct,
|
||||
&password_blob,
|
||||
)])
|
||||
.await
|
||||
.context("InsertBatch for new password")?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn delete_password(
|
||||
storage: &StorageCredentials,
|
||||
password: &str,
|
||||
allow_delete_all: bool,
|
||||
) -> Result<()> {
|
||||
let k2v = storage.k2v_client()?;
|
||||
let (ident_salt, _public) = Self::load_salt_and_public(&k2v).await?;
|
||||
|
||||
// Generate short password digest (= password identity)
|
||||
let ident = argon2_kdf(&ident_salt, password.as_bytes(), 16)?;
|
||||
let password_sortkey = format!("password:{}", hex::encode(&ident));
|
||||
|
||||
// List existing passwords
|
||||
let existing_passwords = Self::list_existing_passwords(&k2v).await?;
|
||||
|
||||
// Check password is there
|
||||
let pw = existing_passwords
|
||||
.get(&password_sortkey)
|
||||
.ok_or(anyhow!("password does not exist"))?;
|
||||
|
||||
if !allow_delete_all && existing_passwords.len() < 2 {
|
||||
bail!("No other password exists, not deleting last password.");
|
||||
}
|
||||
|
||||
k2v.delete_item("keys", &password_sortkey, pw.causality.clone())
|
||||
.await
|
||||
.context("DeleteItem for password")?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---- STORAGE UTIL ----
|
||||
|
||||
async fn check_uninitialized(
|
||||
k2v: &K2vClient,
|
||||
) -> Result<(Option<CausalityToken>, Option<CausalityToken>)> {
|
||||
let params = k2v
|
||||
.read_batch(&[
|
||||
k2v_read_single_key("keys", "salt", true),
|
||||
k2v_read_single_key("keys", "public", true),
|
||||
])
|
||||
.await
|
||||
.context("ReadBatch for salt and public in check_uninitialized")?;
|
||||
if params.len() != 2 {
|
||||
bail!(
|
||||
"Invalid response from k2v storage: {:?} (expected two items)",
|
||||
params
|
||||
);
|
||||
}
|
||||
if params[0].items.len() > 1 || params[1].items.len() > 1 {
|
||||
bail!(
|
||||
"invalid response from k2v storage: {:?} (several items in single_item read)",
|
||||
params
|
||||
);
|
||||
}
|
||||
|
||||
let salt_ct = match params[0].items.iter().next() {
|
||||
None => None,
|
||||
Some((_, CausalValue { causality, value })) => {
|
||||
if value.iter().any(|x| matches!(x, K2vValue::Value(_))) {
|
||||
bail!("key storage already initialized");
|
||||
}
|
||||
Some(causality.clone())
|
||||
}
|
||||
};
|
||||
|
||||
let public_ct = match params[1].items.iter().next() {
|
||||
None => None,
|
||||
Some((_, CausalValue { causality, value })) => {
|
||||
if value.iter().any(|x| matches!(x, K2vValue::Value(_))) {
|
||||
bail!("key storage already initialized");
|
||||
}
|
||||
Some(causality.clone())
|
||||
}
|
||||
};
|
||||
|
||||
Ok((salt_ct, public_ct))
|
||||
}
|
||||
|
||||
pub async fn load_salt_and_public(k2v: &K2vClient) -> Result<([u8; 32], PublicKey)> {
|
||||
let mut params = k2v
|
||||
.read_batch(&[
|
||||
k2v_read_single_key("keys", "salt", false),
|
||||
k2v_read_single_key("keys", "public", false),
|
||||
])
|
||||
.await
|
||||
.context("ReadBatch for salt and public in load_salt_and_public")?;
|
||||
if params.len() != 2 {
|
||||
bail!(
|
||||
"Invalid response from k2v storage: {:?} (expected two items)",
|
||||
params
|
||||
);
|
||||
}
|
||||
if params[0].items.len() != 1 || params[1].items.len() != 1 {
|
||||
bail!("cryptographic keys not initialized for user");
|
||||
}
|
||||
|
||||
// Retrieve salt from given response
|
||||
let salt_vals = &mut params[0].items.iter_mut().next().unwrap().1.value;
|
||||
if salt_vals.len() != 1 {
|
||||
bail!("Multiple values for `salt`");
|
||||
}
|
||||
let salt: Vec<u8> = match &mut salt_vals[0] {
|
||||
K2vValue::Value(v) => std::mem::take(v),
|
||||
K2vValue::Tombstone => bail!("salt is a tombstone"),
|
||||
};
|
||||
if salt.len() != 32 {
|
||||
bail!("`salt` is not 32 bytes long");
|
||||
}
|
||||
let mut salt_constlen = [0u8; 32];
|
||||
salt_constlen.copy_from_slice(&salt);
|
||||
|
||||
// Retrieve public from given response
|
||||
let public_vals = &mut params[1].items.iter_mut().next().unwrap().1.value;
|
||||
if public_vals.len() != 1 {
|
||||
bail!("Multiple values for `public`");
|
||||
}
|
||||
let public: Vec<u8> = match &mut public_vals[0] {
|
||||
K2vValue::Value(v) => std::mem::take(v),
|
||||
K2vValue::Tombstone => bail!("public is a tombstone"),
|
||||
};
|
||||
let public = PublicKey::from_slice(&public).ok_or(anyhow!("Invalid public key length"))?;
|
||||
|
||||
Ok((salt_constlen, public))
|
||||
}
|
||||
|
||||
async fn list_existing_passwords(k2v: &K2vClient) -> Result<BTreeMap<String, CausalValue>> {
|
||||
let mut res = k2v
|
||||
.read_batch(&[BatchReadOp {
|
||||
partition_key: "keys",
|
||||
filter: Filter {
|
||||
start: None,
|
||||
end: None,
|
||||
prefix: Some("password:"),
|
||||
limit: None,
|
||||
reverse: false,
|
||||
},
|
||||
conflicts_only: false,
|
||||
tombstones: false,
|
||||
single_item: false,
|
||||
}])
|
||||
.await
|
||||
.context("ReadBatch for prefix password: in list_existing_passwords")?;
|
||||
if res.len() != 1 {
|
||||
bail!("unexpected k2v result: {:?}, expected one item", res);
|
||||
}
|
||||
Ok(res.pop().unwrap().items)
|
||||
}
|
||||
|
||||
// Clear text serialize/deserialize
|
||||
/// Serialize the root as bytes without encryption
|
||||
fn serialize(&self) -> [u8; 64] {
|
||||
let mut res = [0u8; 64];
|
||||
res[..32].copy_from_slice(self.master.as_ref());
|
||||
|
|
@ -166,7 +488,6 @@ impl CryptoKeys {
|
|||
res
|
||||
}
|
||||
|
||||
/// Deserialize a clear text crypto root without encryption
|
||||
fn deserialize(bytes: &[u8]) -> Result<Self> {
|
||||
if bytes.len() != 64 {
|
||||
bail!("Invalid length: {}, expected 64", bytes.len());
|
||||
|
|
@ -180,66 +501,91 @@ impl CryptoKeys {
|
|||
public,
|
||||
})
|
||||
}
|
||||
|
||||
// Password sealed keys serialize/deserialize
|
||||
pub fn password_open(password: &str, blob: &[u8]) -> Result<Self> {
|
||||
let _pubkey = &blob[0..32];
|
||||
let kdf_salt = &blob[32..64];
|
||||
let password_openned = try_open_encrypted_keys(kdf_salt, password, &blob[64..])?;
|
||||
|
||||
let keys = Self::deserialize(&password_openned)?;
|
||||
Ok(keys)
|
||||
}
|
||||
|
||||
pub fn password_seal(&self, password: &str) -> Result<Vec<u8>> {
|
||||
let mut kdf_salt = [0u8; 32];
|
||||
thread_rng().fill(&mut kdf_salt);
|
||||
|
||||
// Calculate key for password secret box
|
||||
let password_key = derive_password_key(&kdf_salt, password)?;
|
||||
|
||||
// Seal a secret box that contains our crypto keys
|
||||
let password_sealed = seal(&self.serialize(), &password_key)?;
|
||||
|
||||
// Create blob
|
||||
let password_blob = [&self.public[..], &kdf_salt[..], &password_sealed].concat();
|
||||
|
||||
Ok(password_blob)
|
||||
}
|
||||
}
|
||||
|
||||
fn derive_password_key(kdf_salt: &[u8], password: &str) -> Result<Key> {
|
||||
Ok(Key::from_slice(&argon2_kdf(kdf_salt, password.as_bytes(), 32)?).unwrap())
|
||||
}
|
||||
impl UserSecrets {
|
||||
fn derive_password_key_with(user_secret: &str, kdf_salt: &[u8], password: &str) -> Result<Key> {
|
||||
let tmp = format!("{}\n\n{}", user_secret, password);
|
||||
Ok(Key::from_slice(&argon2_kdf(&kdf_salt, tmp.as_bytes(), 32)?).unwrap())
|
||||
}
|
||||
|
||||
fn try_open_encrypted_keys(
|
||||
kdf_salt: &[u8],
|
||||
password: &str,
|
||||
encrypted_keys: &[u8],
|
||||
) -> Result<Vec<u8>> {
|
||||
let password_key = derive_password_key(kdf_salt, password)?;
|
||||
open(encrypted_keys, &password_key)
|
||||
fn derive_password_key(&self, kdf_salt: &[u8], password: &str) -> Result<Key> {
|
||||
Self::derive_password_key_with(&self.user_secret, kdf_salt, password)
|
||||
}
|
||||
|
||||
fn try_open_encrypted_keys(
|
||||
&self,
|
||||
kdf_salt: &[u8],
|
||||
password: &str,
|
||||
encrypted_keys: &[u8],
|
||||
) -> Result<Vec<u8>> {
|
||||
let secrets_to_try =
|
||||
std::iter::once(&self.user_secret).chain(self.alternate_user_secrets.iter());
|
||||
for user_secret in secrets_to_try {
|
||||
let password_key = Self::derive_password_key_with(user_secret, kdf_salt, password)?;
|
||||
if let Ok(res) = open(encrypted_keys, &password_key) {
|
||||
return Ok(res);
|
||||
}
|
||||
}
|
||||
bail!("Unable to decrypt password blob.");
|
||||
}
|
||||
}
|
||||
|
||||
// ---- UTIL ----
|
||||
|
||||
pub fn argon2_kdf(salt: &[u8], password: &[u8], output_len: usize) -> Result<Vec<u8>> {
|
||||
use argon2::{password_hash, Algorithm, Argon2, ParamsBuilder, PasswordHasher, Version};
|
||||
use argon2::{Algorithm, Argon2, ParamsBuilder, PasswordHasher, Version};
|
||||
|
||||
let params = ParamsBuilder::new()
|
||||
let mut params = ParamsBuilder::new();
|
||||
params
|
||||
.output_len(output_len)
|
||||
.build()
|
||||
.map_err(|e| anyhow!("Invalid output length: {}", e))?;
|
||||
|
||||
let params = params
|
||||
.params()
|
||||
.map_err(|e| anyhow!("Invalid argon2 params: {}", e))?;
|
||||
let argon2 = Argon2::new(Algorithm::default(), Version::default(), params);
|
||||
|
||||
let b64_salt = base64::engine::general_purpose::STANDARD_NO_PAD.encode(salt);
|
||||
let valid_salt = password_hash::Salt::from_b64(&b64_salt)
|
||||
.map_err(|e| anyhow!("Invalid salt, error {}", e))?;
|
||||
let salt = base64::encode_config(salt, base64::STANDARD_NO_PAD);
|
||||
let hash = argon2
|
||||
.hash_password(password, valid_salt)
|
||||
.hash_password(password, &salt)
|
||||
.map_err(|e| anyhow!("Unable to hash: {}", e))?;
|
||||
|
||||
let hash = hash.hash.ok_or(anyhow!("Missing output"))?;
|
||||
assert!(hash.len() == output_len);
|
||||
Ok(hash.as_bytes().to_vec())
|
||||
}
|
||||
|
||||
pub fn k2v_read_single_key<'a>(
|
||||
partition_key: &'a str,
|
||||
sort_key: &'a str,
|
||||
tombstones: bool,
|
||||
) -> BatchReadOp<'a> {
|
||||
BatchReadOp {
|
||||
partition_key: partition_key,
|
||||
filter: Filter {
|
||||
start: Some(sort_key),
|
||||
end: None,
|
||||
prefix: None,
|
||||
limit: None,
|
||||
reverse: false,
|
||||
},
|
||||
conflicts_only: false,
|
||||
tombstones,
|
||||
single_item: true,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn k2v_insert_single_key<'a>(
|
||||
partition_key: &'a str,
|
||||
sort_key: &'a str,
|
||||
causality: Option<CausalityToken>,
|
||||
value: impl AsRef<[u8]>,
|
||||
) -> BatchInsertOp<'a> {
|
||||
BatchInsertOp {
|
||||
partition_key,
|
||||
sort_key,
|
||||
causality,
|
||||
value: K2vValue::Value(value.as_ref().to_vec()),
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,91 +1,46 @@
|
|||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use tokio::signal::unix::{signal, SignalKind};
|
||||
use tokio::sync::watch;
|
||||
|
||||
use anyhow::{anyhow, bail, Result};
|
||||
use async_trait::async_trait;
|
||||
use rusoto_signature::Region;
|
||||
|
||||
use crate::config::*;
|
||||
use crate::cryptoblob::{Key, SecretKey};
|
||||
use crate::login::*;
|
||||
use crate::storage;
|
||||
|
||||
pub struct ContextualUserEntry {
|
||||
pub username: String,
|
||||
pub config: UserEntry,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct UserDatabase {
|
||||
users: HashMap<String, Arc<ContextualUserEntry>>,
|
||||
users_by_email: HashMap<String, Arc<ContextualUserEntry>>,
|
||||
}
|
||||
|
||||
pub struct StaticLoginProvider {
|
||||
user_db: watch::Receiver<UserDatabase>,
|
||||
in_memory_store: storage::in_memory::MemDb,
|
||||
garage_store: storage::garage::GarageRoot,
|
||||
default_bucket: Option<String>,
|
||||
users: HashMap<String, Arc<LoginStaticUser>>,
|
||||
users_by_email: HashMap<String, Arc<LoginStaticUser>>,
|
||||
|
||||
k2v_region: Region,
|
||||
s3_region: Region,
|
||||
}
|
||||
|
||||
pub async fn update_user_list(config: PathBuf, up: watch::Sender<UserDatabase>) -> Result<()> {
|
||||
let mut stream = signal(SignalKind::user_defined1())
|
||||
.expect("failed to install SIGUSR1 signal hander for reload");
|
||||
|
||||
loop {
|
||||
let ulist: UserList = match read_config(config.clone()) {
|
||||
Ok(x) => x,
|
||||
Err(e) => {
|
||||
tracing::warn!(path=%config.as_path().to_string_lossy(), error=%e, "Unable to load config");
|
||||
stream.recv().await;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let users = ulist
|
||||
impl StaticLoginProvider {
|
||||
pub fn new(config: LoginStaticConfig, k2v_region: Region, s3_region: Region) -> Result<Self> {
|
||||
let users = config
|
||||
.users
|
||||
.into_iter()
|
||||
.map(|(username, config)| {
|
||||
(
|
||||
username.clone(),
|
||||
Arc::new(ContextualUserEntry { username, config }),
|
||||
)
|
||||
})
|
||||
.map(|(k, v)| (k, Arc::new(v)))
|
||||
.collect::<HashMap<_, _>>();
|
||||
|
||||
let mut users_by_email = HashMap::new();
|
||||
for (_, u) in users.iter() {
|
||||
for m in u.config.email_addresses.iter() {
|
||||
for m in u.email_addresses.iter() {
|
||||
if users_by_email.contains_key(m) {
|
||||
tracing::warn!("Several users have the same email address: {}", m);
|
||||
stream.recv().await;
|
||||
continue;
|
||||
bail!("Several users have same email address: {}", m);
|
||||
}
|
||||
users_by_email.insert(m.clone(), u.clone());
|
||||
}
|
||||
}
|
||||
|
||||
tracing::info!("{} users loaded", users.len());
|
||||
up.send(UserDatabase {
|
||||
Ok(Self {
|
||||
default_bucket: config.default_bucket,
|
||||
users,
|
||||
users_by_email,
|
||||
})
|
||||
.context("update user db config")?;
|
||||
stream.recv().await;
|
||||
tracing::info!("Received SIGUSR1, reloading");
|
||||
}
|
||||
}
|
||||
|
||||
impl StaticLoginProvider {
|
||||
pub async fn new(config: LoginStaticConfig) -> Result<Self> {
|
||||
let (tx, mut rx) = watch::channel(UserDatabase::default());
|
||||
|
||||
tokio::spawn(update_user_list(config.user_list, tx));
|
||||
rx.changed().await?;
|
||||
|
||||
Ok(Self {
|
||||
user_db: rx,
|
||||
in_memory_store: storage::in_memory::MemDb::new(),
|
||||
garage_store: storage::garage::GarageRoot::new()?,
|
||||
k2v_region,
|
||||
s3_region,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -94,67 +49,82 @@ impl StaticLoginProvider {
|
|||
impl LoginProvider for StaticLoginProvider {
|
||||
async fn login(&self, username: &str, password: &str) -> Result<Credentials> {
|
||||
tracing::debug!(user=%username, "login");
|
||||
let user = {
|
||||
let user_db = self.user_db.borrow();
|
||||
match user_db.users.get(username) {
|
||||
None => bail!("User {} does not exist", username),
|
||||
Some(u) => u.clone(),
|
||||
}
|
||||
let user = match self.users.get(username) {
|
||||
None => bail!("User {} does not exist", username),
|
||||
Some(u) => u,
|
||||
};
|
||||
|
||||
tracing::debug!(user=%username, "verify password");
|
||||
if !verify_password(password, &user.config.password)? {
|
||||
if !verify_password(password, &user.password)? {
|
||||
bail!("Wrong password");
|
||||
}
|
||||
|
||||
tracing::debug!(user=%username, "fetch bucket");
|
||||
let bucket = user
|
||||
.bucket
|
||||
.clone()
|
||||
.or_else(|| self.default_bucket.clone())
|
||||
.ok_or(anyhow!(
|
||||
"No bucket configured and no default bucket specieid"
|
||||
))?;
|
||||
|
||||
tracing::debug!(user=%username, "fetch keys");
|
||||
let storage: storage::Builder = match &user.config.storage {
|
||||
StaticStorage::InMemory => self.in_memory_store.builder(username).await,
|
||||
StaticStorage::Garage(grgconf) => {
|
||||
self.garage_store.user(storage::garage::GarageConf {
|
||||
region: grgconf.aws_region.clone(),
|
||||
k2v_endpoint: grgconf.k2v_endpoint.clone(),
|
||||
s3_endpoint: grgconf.s3_endpoint.clone(),
|
||||
aws_access_key_id: grgconf.aws_access_key_id.clone(),
|
||||
aws_secret_access_key: grgconf.aws_secret_access_key.clone(),
|
||||
bucket: grgconf.bucket.clone(),
|
||||
})?
|
||||
}
|
||||
let storage = StorageCredentials {
|
||||
k2v_region: self.k2v_region.clone(),
|
||||
s3_region: self.s3_region.clone(),
|
||||
aws_access_key_id: user.aws_access_key_id.clone(),
|
||||
aws_secret_access_key: user.aws_secret_access_key.clone(),
|
||||
bucket,
|
||||
};
|
||||
|
||||
let cr = CryptoRoot(user.config.crypto_root.clone());
|
||||
let keys = cr.crypto_keys(password)?;
|
||||
let keys = match (&user.master_key, &user.secret_key) {
|
||||
(Some(m), Some(s)) => {
|
||||
let master_key =
|
||||
Key::from_slice(&base64::decode(m)?).ok_or(anyhow!("Invalid master key"))?;
|
||||
let secret_key = SecretKey::from_slice(&base64::decode(s)?)
|
||||
.ok_or(anyhow!("Invalid secret key"))?;
|
||||
CryptoKeys::open_without_password(&storage, &master_key, &secret_key).await?
|
||||
}
|
||||
(None, None) => {
|
||||
let user_secrets = UserSecrets {
|
||||
user_secret: user.user_secret.clone(),
|
||||
alternate_user_secrets: user.alternate_user_secrets.clone(),
|
||||
};
|
||||
CryptoKeys::open(&storage, &user_secrets, password).await?
|
||||
}
|
||||
_ => bail!(
|
||||
"Either both master and secret key or none of them must be specified for user"
|
||||
),
|
||||
};
|
||||
|
||||
tracing::debug!(user=%username, "logged");
|
||||
Ok(Credentials { storage, keys })
|
||||
}
|
||||
|
||||
async fn public_login(&self, email: &str) -> Result<PublicCredentials> {
|
||||
let user = {
|
||||
let user_db = self.user_db.borrow();
|
||||
match user_db.users_by_email.get(email) {
|
||||
None => bail!("Email {} does not exist", email),
|
||||
Some(u) => u.clone(),
|
||||
}
|
||||
};
|
||||
tracing::debug!(user=%user.username, "public_login");
|
||||
|
||||
let storage: storage::Builder = match &user.config.storage {
|
||||
StaticStorage::InMemory => self.in_memory_store.builder(&user.username).await,
|
||||
StaticStorage::Garage(grgconf) => {
|
||||
self.garage_store.user(storage::garage::GarageConf {
|
||||
region: grgconf.aws_region.clone(),
|
||||
k2v_endpoint: grgconf.k2v_endpoint.clone(),
|
||||
s3_endpoint: grgconf.s3_endpoint.clone(),
|
||||
aws_access_key_id: grgconf.aws_access_key_id.clone(),
|
||||
aws_secret_access_key: grgconf.aws_secret_access_key.clone(),
|
||||
bucket: grgconf.bucket.clone(),
|
||||
})?
|
||||
}
|
||||
let user = match self.users_by_email.get(email) {
|
||||
None => bail!("No user for email address {}", email),
|
||||
Some(u) => u,
|
||||
};
|
||||
|
||||
let cr = CryptoRoot(user.config.crypto_root.clone());
|
||||
let public_key = cr.public_key()?;
|
||||
let bucket = user
|
||||
.bucket
|
||||
.clone()
|
||||
.or_else(|| self.default_bucket.clone())
|
||||
.ok_or(anyhow!(
|
||||
"No bucket configured and no default bucket specieid"
|
||||
))?;
|
||||
|
||||
let storage = StorageCredentials {
|
||||
k2v_region: self.k2v_region.clone(),
|
||||
s3_region: self.s3_region.clone(),
|
||||
aws_access_key_id: user.aws_access_key_id.clone(),
|
||||
aws_secret_access_key: user.aws_secret_access_key.clone(),
|
||||
bucket,
|
||||
};
|
||||
|
||||
let k2v_client = storage.k2v_client()?;
|
||||
let (_, public_key) = CryptoKeys::load_salt_and_public(&k2v_client).await?;
|
||||
|
||||
Ok(PublicCredentials {
|
||||
storage,
|
||||
|
|
@ -182,7 +152,7 @@ pub fn verify_password(password: &str, hash: &str) -> Result<bool> {
|
|||
Argon2,
|
||||
};
|
||||
let parsed_hash =
|
||||
PasswordHash::new(hash).map_err(|e| anyhow!("Invalid hashed password: {}", e))?;
|
||||
PasswordHash::new(&hash).map_err(|e| anyhow!("Invalid hashed password: {}", e))?;
|
||||
Ok(Argon2::default()
|
||||
.verify_password(password.as_bytes(), &parsed_hash)
|
||||
.is_ok())
|
||||
|
|
|
|||
|
|
@ -1,445 +0,0 @@
|
|||
//use std::collections::HashMap;
|
||||
use std::convert::TryFrom;
|
||||
|
||||
use std::sync::{Arc, Weak};
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{anyhow, bail, Result};
|
||||
use base64::Engine;
|
||||
use futures::{future::BoxFuture, FutureExt};
|
||||
//use tokio::io::AsyncReadExt;
|
||||
use tokio::sync::watch;
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
use crate::cryptoblob;
|
||||
use crate::login::{Credentials, PublicCredentials};
|
||||
use crate::mail::mailbox::Mailbox;
|
||||
use crate::mail::uidindex::ImapUidvalidity;
|
||||
use crate::mail::unique_ident::*;
|
||||
use crate::mail::user::User;
|
||||
use crate::mail::IMF;
|
||||
use crate::storage;
|
||||
use crate::timestamp::now_msec;
|
||||
|
||||
const INCOMING_PK: &str = "incoming";
|
||||
const INCOMING_LOCK_SK: &str = "lock";
|
||||
const INCOMING_WATCH_SK: &str = "watch";
|
||||
|
||||
const MESSAGE_KEY: &str = "message-key";
|
||||
|
||||
// When a lock is held, it is held for LOCK_DURATION (here 5 minutes)
|
||||
// It is renewed every LOCK_DURATION/3
|
||||
// If we are at 2*LOCK_DURATION/3 and haven't renewed, we assume we
|
||||
// lost the lock.
|
||||
const LOCK_DURATION: Duration = Duration::from_secs(300);
|
||||
|
||||
// In addition to checking when notified, also check for new mail every 10 minutes
|
||||
const MAIL_CHECK_INTERVAL: Duration = Duration::from_secs(600);
|
||||
|
||||
pub async fn incoming_mail_watch_process(
|
||||
user: Weak<User>,
|
||||
creds: Credentials,
|
||||
rx_inbox_id: watch::Receiver<Option<(UniqueIdent, ImapUidvalidity)>>,
|
||||
) {
|
||||
if let Err(e) = incoming_mail_watch_process_internal(user, creds, rx_inbox_id).await {
|
||||
error!("Error in incoming mail watch process: {}", e);
|
||||
}
|
||||
}
|
||||
|
||||
async fn incoming_mail_watch_process_internal(
|
||||
user: Weak<User>,
|
||||
creds: Credentials,
|
||||
mut rx_inbox_id: watch::Receiver<Option<(UniqueIdent, ImapUidvalidity)>>,
|
||||
) -> Result<()> {
|
||||
let mut lock_held = k2v_lock_loop(
|
||||
creds.storage.build().await?,
|
||||
storage::RowRef::new(INCOMING_PK, INCOMING_LOCK_SK),
|
||||
);
|
||||
let storage = creds.storage.build().await?;
|
||||
|
||||
let mut inbox: Option<Arc<Mailbox>> = None;
|
||||
let mut incoming_key = storage::RowRef::new(INCOMING_PK, INCOMING_WATCH_SK);
|
||||
|
||||
loop {
|
||||
let maybe_updated_incoming_key = if *lock_held.borrow() {
|
||||
debug!("incoming lock held");
|
||||
|
||||
let wait_new_mail = async {
|
||||
loop {
|
||||
match storage.row_poll(&incoming_key).await {
|
||||
Ok(row_val) => break row_val.row_ref,
|
||||
Err(e) => {
|
||||
error!("Error in wait_new_mail: {}", e);
|
||||
tokio::time::sleep(Duration::from_secs(30)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
tokio::select! {
|
||||
inc_k = wait_new_mail => Some(inc_k),
|
||||
_ = tokio::time::sleep(MAIL_CHECK_INTERVAL) => Some(incoming_key.clone()),
|
||||
_ = lock_held.changed() => None,
|
||||
_ = rx_inbox_id.changed() => None,
|
||||
}
|
||||
} else {
|
||||
debug!("incoming lock not held");
|
||||
tokio::select! {
|
||||
_ = lock_held.changed() => None,
|
||||
_ = rx_inbox_id.changed() => None,
|
||||
}
|
||||
};
|
||||
|
||||
let user = match Weak::upgrade(&user) {
|
||||
Some(user) => user,
|
||||
None => {
|
||||
debug!("User no longer available, exiting incoming loop.");
|
||||
break;
|
||||
}
|
||||
};
|
||||
debug!("User still available");
|
||||
|
||||
// If INBOX no longer is same mailbox, open new mailbox
|
||||
let inbox_id = *rx_inbox_id.borrow();
|
||||
if let Some((id, uidvalidity)) = inbox_id {
|
||||
if Some(id) != inbox.as_ref().map(|b| b.id) {
|
||||
match user.open_mailbox_by_id(id, uidvalidity).await {
|
||||
Ok(mb) => {
|
||||
inbox = Some(mb);
|
||||
}
|
||||
Err(e) => {
|
||||
inbox = None;
|
||||
error!("Error when opening inbox ({}): {}", id, e);
|
||||
tokio::time::sleep(Duration::from_secs(30)).await;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If we were able to open INBOX, and we have mail,
|
||||
// fetch new mail
|
||||
if let (Some(inbox), Some(updated_incoming_key)) = (&inbox, maybe_updated_incoming_key) {
|
||||
match handle_incoming_mail(&user, &storage, inbox, &lock_held).await {
|
||||
Ok(()) => {
|
||||
incoming_key = updated_incoming_key;
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Could not fetch incoming mail: {}", e);
|
||||
tokio::time::sleep(Duration::from_secs(30)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
drop(rx_inbox_id);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn handle_incoming_mail(
|
||||
user: &Arc<User>,
|
||||
storage: &storage::Store,
|
||||
inbox: &Arc<Mailbox>,
|
||||
lock_held: &watch::Receiver<bool>,
|
||||
) -> Result<()> {
|
||||
let mails_res = storage.blob_list("incoming/").await?;
|
||||
|
||||
for object in mails_res {
|
||||
if !*lock_held.borrow() {
|
||||
break;
|
||||
}
|
||||
let key = object.0;
|
||||
if let Some(mail_id) = key.strip_prefix("incoming/") {
|
||||
if let Ok(mail_id) = mail_id.parse::<UniqueIdent>() {
|
||||
move_incoming_message(user, storage, inbox, mail_id).await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn move_incoming_message(
|
||||
user: &Arc<User>,
|
||||
storage: &storage::Store,
|
||||
inbox: &Arc<Mailbox>,
|
||||
id: UniqueIdent,
|
||||
) -> Result<()> {
|
||||
info!("Moving incoming message: {}", id);
|
||||
|
||||
let object_key = format!("incoming/{}", id);
|
||||
|
||||
// 1. Fetch message from S3
|
||||
let object = storage.blob_fetch(&storage::BlobRef(object_key)).await?;
|
||||
|
||||
// 1.a decrypt message key from headers
|
||||
//info!("Object metadata: {:?}", get_result.metadata);
|
||||
let key_encrypted_b64 = object
|
||||
.meta
|
||||
.get(MESSAGE_KEY)
|
||||
.ok_or(anyhow!("Missing key in metadata"))?;
|
||||
let key_encrypted = base64::engine::general_purpose::STANDARD.decode(key_encrypted_b64)?;
|
||||
let message_key = sodiumoxide::crypto::sealedbox::open(
|
||||
&key_encrypted,
|
||||
&user.creds.keys.public,
|
||||
&user.creds.keys.secret,
|
||||
)
|
||||
.map_err(|_| anyhow!("Cannot decrypt message key"))?;
|
||||
let message_key =
|
||||
cryptoblob::Key::from_slice(&message_key).ok_or(anyhow!("Invalid message key"))?;
|
||||
|
||||
// 1.b retrieve message body
|
||||
let obj_body = object.value;
|
||||
let plain_mail = cryptoblob::open(&obj_body, &message_key)
|
||||
.map_err(|_| anyhow!("Cannot decrypt email content"))?;
|
||||
|
||||
// 2 parse mail and add to inbox
|
||||
let msg = IMF::try_from(&plain_mail[..]).map_err(|_| anyhow!("Invalid email body"))?;
|
||||
inbox
|
||||
.append_from_s3(msg, id, object.blob_ref.clone(), message_key)
|
||||
.await?;
|
||||
|
||||
// 3 delete from incoming
|
||||
storage.blob_rm(&object.blob_ref).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---- UTIL: K2V locking loop, use this to try to grab a lock using a K2V entry as a signal ----
|
||||
|
||||
fn k2v_lock_loop(storage: storage::Store, row_ref: storage::RowRef) -> watch::Receiver<bool> {
|
||||
let (held_tx, held_rx) = watch::channel(false);
|
||||
|
||||
tokio::spawn(k2v_lock_loop_internal(storage, row_ref, held_tx));
|
||||
|
||||
held_rx
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
enum LockState {
|
||||
Unknown,
|
||||
Empty,
|
||||
Held(UniqueIdent, u64, storage::RowRef),
|
||||
}
|
||||
|
||||
async fn k2v_lock_loop_internal(
|
||||
storage: storage::Store,
|
||||
row_ref: storage::RowRef,
|
||||
held_tx: watch::Sender<bool>,
|
||||
) {
|
||||
let (state_tx, mut state_rx) = watch::channel::<LockState>(LockState::Unknown);
|
||||
let mut state_rx_2 = state_rx.clone();
|
||||
|
||||
let our_pid = gen_ident();
|
||||
|
||||
// Loop 1: watch state of lock in K2V, save that in corresponding watch channel
|
||||
let watch_lock_loop: BoxFuture<Result<()>> = async {
|
||||
let mut ct = row_ref.clone();
|
||||
loop {
|
||||
debug!("k2v watch lock loop iter: ct = {:?}", ct);
|
||||
match storage.row_poll(&ct).await {
|
||||
Err(e) => {
|
||||
error!(
|
||||
"Error in k2v wait value changed: {} ; assuming we no longer hold lock.",
|
||||
e
|
||||
);
|
||||
state_tx.send(LockState::Unknown)?;
|
||||
tokio::time::sleep(Duration::from_secs(30)).await;
|
||||
}
|
||||
Ok(cv) => {
|
||||
let mut lock_state = None;
|
||||
for v in cv.value.iter() {
|
||||
if let storage::Alternative::Value(vbytes) = v {
|
||||
if vbytes.len() == 32 {
|
||||
let ts = u64::from_be_bytes(vbytes[..8].try_into().unwrap());
|
||||
let pid = UniqueIdent(vbytes[8..].try_into().unwrap());
|
||||
if lock_state
|
||||
.map(|(pid2, ts2)| ts > ts2 || (ts == ts2 && pid > pid2))
|
||||
.unwrap_or(true)
|
||||
{
|
||||
lock_state = Some((pid, ts));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let new_ct = cv.row_ref;
|
||||
|
||||
debug!(
|
||||
"k2v watch lock loop: changed, old ct = {:?}, new ct = {:?}, v = {:?}",
|
||||
ct, new_ct, lock_state
|
||||
);
|
||||
state_tx.send(
|
||||
lock_state
|
||||
.map(|(pid, ts)| LockState::Held(pid, ts, new_ct.clone()))
|
||||
.unwrap_or(LockState::Empty),
|
||||
)?;
|
||||
ct = new_ct;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
.boxed();
|
||||
|
||||
// Loop 2: notify user whether we are holding the lock or not
|
||||
let lock_notify_loop: BoxFuture<Result<()>> = async {
|
||||
loop {
|
||||
let now = now_msec();
|
||||
let held_with_expiration_time = match &*state_rx.borrow_and_update() {
|
||||
LockState::Held(pid, ts, _ct) if *pid == our_pid => {
|
||||
let expiration_time = *ts - (LOCK_DURATION / 3).as_millis() as u64;
|
||||
if now < expiration_time {
|
||||
Some(expiration_time)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let held = held_with_expiration_time.is_some();
|
||||
if held != *held_tx.borrow() {
|
||||
held_tx.send(held)?;
|
||||
}
|
||||
|
||||
let await_expired = async {
|
||||
match held_with_expiration_time {
|
||||
None => futures::future::pending().await,
|
||||
Some(expiration_time) => {
|
||||
tokio::time::sleep(Duration::from_millis(expiration_time - now)).await
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
tokio::select!(
|
||||
r = state_rx.changed() => {
|
||||
r?;
|
||||
}
|
||||
_ = held_tx.closed() => bail!("held_tx closed, don't need to hold lock anymore"),
|
||||
_ = await_expired => continue,
|
||||
);
|
||||
}
|
||||
}
|
||||
.boxed();
|
||||
|
||||
// Loop 3: acquire lock when relevant
|
||||
let take_lock_loop: BoxFuture<Result<()>> = async {
|
||||
loop {
|
||||
let now = now_msec();
|
||||
let state: LockState = state_rx_2.borrow_and_update().clone();
|
||||
let (acquire_at, ct) = match state {
|
||||
LockState::Unknown => {
|
||||
// If state of the lock is unknown, don't try to acquire
|
||||
state_rx_2.changed().await?;
|
||||
continue;
|
||||
}
|
||||
LockState::Empty => (now, None),
|
||||
LockState::Held(pid, ts, ct) => {
|
||||
if pid == our_pid {
|
||||
(ts - (2 * LOCK_DURATION / 3).as_millis() as u64, Some(ct))
|
||||
} else {
|
||||
(ts, Some(ct))
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Wait until it is time to acquire lock
|
||||
if acquire_at > now {
|
||||
tokio::select!(
|
||||
r = state_rx_2.changed() => {
|
||||
// If lock state changed in the meantime, don't acquire and loop around
|
||||
r?;
|
||||
continue;
|
||||
}
|
||||
_ = tokio::time::sleep(Duration::from_millis(acquire_at - now)) => ()
|
||||
);
|
||||
}
|
||||
|
||||
// Acquire lock
|
||||
let mut lock = vec![0u8; 32];
|
||||
lock[..8].copy_from_slice(&u64::to_be_bytes(
|
||||
now_msec() + LOCK_DURATION.as_millis() as u64,
|
||||
));
|
||||
lock[8..].copy_from_slice(&our_pid.0);
|
||||
let row = match ct {
|
||||
Some(existing) => existing,
|
||||
None => row_ref.clone(),
|
||||
};
|
||||
if let Err(e) = storage
|
||||
.row_insert(vec![storage::RowVal::new(row, lock)])
|
||||
.await
|
||||
{
|
||||
error!("Could not take lock: {}", e);
|
||||
tokio::time::sleep(Duration::from_secs(30)).await;
|
||||
}
|
||||
|
||||
// Wait for new information to loop back
|
||||
state_rx_2.changed().await?;
|
||||
}
|
||||
}
|
||||
.boxed();
|
||||
|
||||
let _ = futures::try_join!(watch_lock_loop, lock_notify_loop, take_lock_loop);
|
||||
|
||||
debug!("lock loop exited, releasing");
|
||||
|
||||
if !held_tx.is_closed() {
|
||||
warn!("weird...");
|
||||
let _ = held_tx.send(false);
|
||||
}
|
||||
|
||||
// If lock is ours, release it
|
||||
let release = match &*state_rx.borrow() {
|
||||
LockState::Held(pid, _, ct) if *pid == our_pid => Some(ct.clone()),
|
||||
_ => None,
|
||||
};
|
||||
if let Some(ct) = release {
|
||||
match storage.row_rm(&storage::Selector::Single(&ct)).await {
|
||||
Err(e) => warn!("Unable to release lock {:?}: {}", ct, e),
|
||||
Ok(_) => (),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ---- LMTP SIDE: storing messages encrypted with user's pubkey ----
|
||||
|
||||
pub struct EncryptedMessage {
|
||||
key: cryptoblob::Key,
|
||||
encrypted_body: Vec<u8>,
|
||||
}
|
||||
|
||||
impl EncryptedMessage {
|
||||
pub fn new(body: Vec<u8>) -> Result<Self> {
|
||||
let key = cryptoblob::gen_key();
|
||||
let encrypted_body = cryptoblob::seal(&body, &key)?;
|
||||
Ok(Self {
|
||||
key,
|
||||
encrypted_body,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn deliver_to(self: Arc<Self>, creds: PublicCredentials) -> Result<()> {
|
||||
let storage = creds.storage.build().await?;
|
||||
|
||||
// Get causality token of previous watch key
|
||||
let query = storage::RowRef::new(INCOMING_PK, INCOMING_WATCH_SK);
|
||||
let watch_ct = match storage.row_fetch(&storage::Selector::Single(&query)).await {
|
||||
Err(_) => query,
|
||||
Ok(cv) => cv.into_iter().next().map(|v| v.row_ref).unwrap_or(query),
|
||||
};
|
||||
|
||||
// Write mail to encrypted storage
|
||||
let encrypted_key =
|
||||
sodiumoxide::crypto::sealedbox::seal(self.key.as_ref(), &creds.public_key);
|
||||
let key_header = base64::engine::general_purpose::STANDARD.encode(&encrypted_key);
|
||||
|
||||
let blob_val = storage::BlobVal::new(
|
||||
storage::BlobRef(format!("incoming/{}", gen_ident())),
|
||||
self.encrypted_body.clone().into(),
|
||||
)
|
||||
.with_meta(MESSAGE_KEY.to_string(), key_header);
|
||||
storage.blob_insert(blob_val).await?;
|
||||
|
||||
// Update watch key to signal new mail
|
||||
let watch_val = storage::RowVal::new(watch_ct.clone(), gen_ident().0.to_vec());
|
||||
storage.row_insert(vec![watch_val]).await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
@ -5,7 +5,7 @@ use lazy_static::lazy_static;
|
|||
use rand::prelude::*;
|
||||
use serde::{de::Error, Deserialize, Deserializer, Serialize, Serializer};
|
||||
|
||||
use crate::timestamp::now_msec;
|
||||
use crate::time::now_msec;
|
||||
|
||||
/// An internal Mail Identifier is composed of two components:
|
||||
/// - a process identifier, 128 bits, itself composed of:
|
||||
|
|
@ -13,11 +13,11 @@ use crate::timestamp::now_msec;
|
|||
/// - a 64-bit random number
|
||||
/// - a sequence number, 64 bits
|
||||
/// They are not part of the protocol but an internal representation
|
||||
/// required by Aerogramme.
|
||||
/// required by Mailrage/Aerogramme.
|
||||
/// Their main property is to be unique without having to rely
|
||||
/// on synchronization between IMAP processes.
|
||||
#[derive(Clone, Copy, PartialOrd, Ord, PartialEq, Eq, Hash)]
|
||||
pub struct UniqueIdent(pub [u8; 24]);
|
||||
#[derive(Clone, Copy, PartialOrd, Ord, PartialEq, Eq, Hash, Debug)]
|
||||
pub struct MailIdent(pub [u8; 24]);
|
||||
|
||||
struct IdentGenerator {
|
||||
pid: u128,
|
||||
|
|
@ -34,12 +34,12 @@ impl IdentGenerator {
|
|||
}
|
||||
}
|
||||
|
||||
fn gen(&self) -> UniqueIdent {
|
||||
fn gen(&self) -> MailIdent {
|
||||
let sn = self.sn.fetch_add(1, Ordering::Relaxed);
|
||||
let mut res = [0u8; 24];
|
||||
res[0..16].copy_from_slice(&u128::to_be_bytes(self.pid));
|
||||
res[16..24].copy_from_slice(&u64::to_be_bytes(sn));
|
||||
UniqueIdent(res)
|
||||
MailIdent(res)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -47,23 +47,23 @@ lazy_static! {
|
|||
static ref GENERATOR: IdentGenerator = IdentGenerator::new();
|
||||
}
|
||||
|
||||
pub fn gen_ident() -> UniqueIdent {
|
||||
pub fn gen_ident() -> MailIdent {
|
||||
GENERATOR.gen()
|
||||
}
|
||||
|
||||
// -- serde --
|
||||
|
||||
impl<'de> Deserialize<'de> for UniqueIdent {
|
||||
impl<'de> Deserialize<'de> for MailIdent {
|
||||
fn deserialize<D>(d: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
let v = String::deserialize(d)?;
|
||||
UniqueIdent::from_str(&v).map_err(D::Error::custom)
|
||||
MailIdent::from_str(&v).map_err(D::Error::custom)
|
||||
}
|
||||
}
|
||||
|
||||
impl Serialize for UniqueIdent {
|
||||
impl Serialize for MailIdent {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: Serializer,
|
||||
|
|
@ -72,22 +72,16 @@ impl Serialize for UniqueIdent {
|
|||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for UniqueIdent {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{}", hex::encode(self.0))
|
||||
impl ToString for MailIdent {
|
||||
fn to_string(&self) -> String {
|
||||
hex::encode(self.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for UniqueIdent {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{}", hex::encode(self.0))
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for UniqueIdent {
|
||||
impl FromStr for MailIdent {
|
||||
type Err = &'static str;
|
||||
|
||||
fn from_str(s: &str) -> Result<UniqueIdent, &'static str> {
|
||||
fn from_str(s: &str) -> Result<MailIdent, &'static str> {
|
||||
let bytes = hex::decode(s).map_err(|_| "invalid hex")?;
|
||||
|
||||
if bytes.len() != 24 {
|
||||
|
|
@ -96,6 +90,6 @@ impl FromStr for UniqueIdent {
|
|||
|
||||
let mut tmp = [0u8; 24];
|
||||
tmp[..].copy_from_slice(&bytes);
|
||||
Ok(UniqueIdent(tmp))
|
||||
Ok(MailIdent(tmp))
|
||||
}
|
||||
}
|
||||
|
|
@ -1,524 +0,0 @@
|
|||
use anyhow::{anyhow, bail, Result};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
use crate::bayou::Bayou;
|
||||
use crate::cryptoblob::{self, gen_key, open_deserialize, seal_serialize, Key};
|
||||
use crate::login::Credentials;
|
||||
use crate::mail::uidindex::*;
|
||||
use crate::mail::unique_ident::*;
|
||||
use crate::mail::IMF;
|
||||
use crate::storage::{self, BlobRef, BlobVal, RowRef, RowVal, Selector, Store};
|
||||
use crate::timestamp::now_msec;
|
||||
|
||||
pub struct Mailbox {
|
||||
pub(super) id: UniqueIdent,
|
||||
mbox: RwLock<MailboxInternal>,
|
||||
}
|
||||
|
||||
impl Mailbox {
|
||||
pub(super) async fn open(
|
||||
creds: &Credentials,
|
||||
id: UniqueIdent,
|
||||
min_uidvalidity: ImapUidvalidity,
|
||||
) -> Result<Self> {
|
||||
let index_path = format!("index/{}", id);
|
||||
let mail_path = format!("mail/{}", id);
|
||||
|
||||
let mut uid_index = Bayou::<UidIndex>::new(creds, index_path).await?;
|
||||
uid_index.sync().await?;
|
||||
|
||||
let uidvalidity = uid_index.state().uidvalidity;
|
||||
if uidvalidity < min_uidvalidity {
|
||||
uid_index
|
||||
.push(
|
||||
uid_index
|
||||
.state()
|
||||
.op_bump_uidvalidity(min_uidvalidity.get() - uidvalidity.get()),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
// @FIXME reporting through opentelemetry or some logs
|
||||
// info on the "shape" of the mailbox would be welcomed
|
||||
/*
|
||||
dump(&uid_index);
|
||||
*/
|
||||
|
||||
let mbox = RwLock::new(MailboxInternal {
|
||||
id,
|
||||
encryption_key: creds.keys.master.clone(),
|
||||
storage: creds.storage.build().await?,
|
||||
uid_index,
|
||||
mail_path,
|
||||
});
|
||||
|
||||
Ok(Self { id, mbox })
|
||||
}
|
||||
|
||||
/// Sync data with backing store
|
||||
pub async fn force_sync(&self) -> Result<()> {
|
||||
self.mbox.write().await.force_sync().await
|
||||
}
|
||||
|
||||
/// Sync data with backing store only if changes are detected
|
||||
/// or last sync is too old
|
||||
pub async fn opportunistic_sync(&self) -> Result<()> {
|
||||
self.mbox.write().await.opportunistic_sync().await
|
||||
}
|
||||
|
||||
/// Block until a sync has been done (due to changes in the event log)
|
||||
pub async fn notify(&self) -> std::sync::Weak<tokio::sync::Notify> {
|
||||
self.mbox.read().await.notifier()
|
||||
}
|
||||
|
||||
// ---- Functions for reading the mailbox ----
|
||||
|
||||
/// Get a clone of the current UID Index of this mailbox
|
||||
/// (cloning is cheap so don't hesitate to use this)
|
||||
pub async fn current_uid_index(&self) -> UidIndex {
|
||||
self.mbox.read().await.uid_index.state().clone()
|
||||
}
|
||||
|
||||
/// Fetch the metadata (headers + some more info) of the specified
|
||||
/// mail IDs
|
||||
pub async fn fetch_meta(&self, ids: &[UniqueIdent]) -> Result<Vec<MailMeta>> {
|
||||
self.mbox.read().await.fetch_meta(ids).await
|
||||
}
|
||||
|
||||
/// Fetch an entire e-mail
|
||||
pub async fn fetch_full(&self, id: UniqueIdent, message_key: &Key) -> Result<Vec<u8>> {
|
||||
self.mbox.read().await.fetch_full(id, message_key).await
|
||||
}
|
||||
|
||||
pub async fn frozen(self: &std::sync::Arc<Self>) -> super::snapshot::FrozenMailbox {
|
||||
super::snapshot::FrozenMailbox::new(self.clone()).await
|
||||
}
|
||||
|
||||
// ---- Functions for changing the mailbox ----
|
||||
|
||||
/// Add flags to message
|
||||
pub async fn add_flags<'a>(&self, id: UniqueIdent, flags: &[Flag]) -> Result<()> {
|
||||
self.mbox.write().await.add_flags(id, flags).await
|
||||
}
|
||||
|
||||
/// Delete flags from message
|
||||
pub async fn del_flags<'a>(&self, id: UniqueIdent, flags: &[Flag]) -> Result<()> {
|
||||
self.mbox.write().await.del_flags(id, flags).await
|
||||
}
|
||||
|
||||
/// Define the new flags for this message
|
||||
pub async fn set_flags<'a>(&self, id: UniqueIdent, flags: &[Flag]) -> Result<()> {
|
||||
self.mbox.write().await.set_flags(id, flags).await
|
||||
}
|
||||
|
||||
/// Insert an email into the mailbox
|
||||
pub async fn append<'a>(
|
||||
&self,
|
||||
msg: IMF<'a>,
|
||||
ident: Option<UniqueIdent>,
|
||||
flags: &[Flag],
|
||||
) -> Result<(ImapUidvalidity, ImapUid, ModSeq)> {
|
||||
self.mbox.write().await.append(msg, ident, flags).await
|
||||
}
|
||||
|
||||
/// Insert an email into the mailbox, copying it from an existing S3 object
|
||||
pub async fn append_from_s3<'a>(
|
||||
&self,
|
||||
msg: IMF<'a>,
|
||||
ident: UniqueIdent,
|
||||
blob_ref: storage::BlobRef,
|
||||
message_key: Key,
|
||||
) -> Result<()> {
|
||||
self.mbox
|
||||
.write()
|
||||
.await
|
||||
.append_from_s3(msg, ident, blob_ref, message_key)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Delete a message definitively from the mailbox
|
||||
pub async fn delete<'a>(&self, id: UniqueIdent) -> Result<()> {
|
||||
self.mbox.write().await.delete(id).await
|
||||
}
|
||||
|
||||
/// Copy an email from an other Mailbox to this mailbox
|
||||
/// (use this when possible, as it allows for a certain number of storage optimizations)
|
||||
pub async fn copy_from(&self, from: &Mailbox, uuid: UniqueIdent) -> Result<UniqueIdent> {
|
||||
if self.id == from.id {
|
||||
bail!("Cannot copy into same mailbox");
|
||||
}
|
||||
|
||||
let (mut selflock, fromlock);
|
||||
if self.id < from.id {
|
||||
selflock = self.mbox.write().await;
|
||||
fromlock = from.mbox.write().await;
|
||||
} else {
|
||||
fromlock = from.mbox.write().await;
|
||||
selflock = self.mbox.write().await;
|
||||
};
|
||||
selflock.copy_from(&fromlock, uuid).await
|
||||
}
|
||||
|
||||
/// Move an email from an other Mailbox to this mailbox
|
||||
/// (use this when possible, as it allows for a certain number of storage optimizations)
|
||||
pub async fn move_from(&self, from: &Mailbox, uuid: UniqueIdent) -> Result<()> {
|
||||
if self.id == from.id {
|
||||
bail!("Cannot copy move same mailbox");
|
||||
}
|
||||
|
||||
let (mut selflock, mut fromlock);
|
||||
if self.id < from.id {
|
||||
selflock = self.mbox.write().await;
|
||||
fromlock = from.mbox.write().await;
|
||||
} else {
|
||||
fromlock = from.mbox.write().await;
|
||||
selflock = self.mbox.write().await;
|
||||
};
|
||||
selflock.move_from(&mut fromlock, uuid).await
|
||||
}
|
||||
}
|
||||
|
||||
// ----
|
||||
|
||||
// Non standard but common flags:
|
||||
// https://www.iana.org/assignments/imap-jmap-keywords/imap-jmap-keywords.xhtml
|
||||
struct MailboxInternal {
|
||||
// 2023-05-15 will probably be used later.
|
||||
#[allow(dead_code)]
|
||||
id: UniqueIdent,
|
||||
mail_path: String,
|
||||
encryption_key: Key,
|
||||
storage: Store,
|
||||
uid_index: Bayou<UidIndex>,
|
||||
}
|
||||
|
||||
impl MailboxInternal {
|
||||
async fn force_sync(&mut self) -> Result<()> {
|
||||
self.uid_index.sync().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn opportunistic_sync(&mut self) -> Result<()> {
|
||||
self.uid_index.opportunistic_sync().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn notifier(&self) -> std::sync::Weak<tokio::sync::Notify> {
|
||||
self.uid_index.notifier()
|
||||
}
|
||||
|
||||
// ---- Functions for reading the mailbox ----
|
||||
|
||||
async fn fetch_meta(&self, ids: &[UniqueIdent]) -> Result<Vec<MailMeta>> {
|
||||
let ids = ids.iter().map(|x| x.to_string()).collect::<Vec<_>>();
|
||||
let ops = ids
|
||||
.iter()
|
||||
.map(|id| RowRef::new(self.mail_path.as_str(), id.as_str()))
|
||||
.collect::<Vec<_>>();
|
||||
let res_vec = self.storage.row_fetch(&Selector::List(ops)).await?;
|
||||
|
||||
let mut meta_vec = vec![];
|
||||
for res in res_vec.into_iter() {
|
||||
let mut meta_opt = None;
|
||||
|
||||
// Resolve conflicts
|
||||
for v in res.value.iter() {
|
||||
match v {
|
||||
storage::Alternative::Tombstone => (),
|
||||
storage::Alternative::Value(v) => {
|
||||
let meta = open_deserialize::<MailMeta>(v, &self.encryption_key)?;
|
||||
match meta_opt.as_mut() {
|
||||
None => {
|
||||
meta_opt = Some(meta);
|
||||
}
|
||||
Some(prevmeta) => {
|
||||
prevmeta.try_merge(meta)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(meta) = meta_opt {
|
||||
meta_vec.push(meta);
|
||||
} else {
|
||||
bail!("No valid meta value in k2v for {:?}", res.row_ref);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(meta_vec)
|
||||
}
|
||||
|
||||
async fn fetch_full(&self, id: UniqueIdent, message_key: &Key) -> Result<Vec<u8>> {
|
||||
let obj_res = self
|
||||
.storage
|
||||
.blob_fetch(&BlobRef(format!("{}/{}", self.mail_path, id)))
|
||||
.await?;
|
||||
let body = obj_res.value;
|
||||
cryptoblob::open(&body, message_key)
|
||||
}
|
||||
|
||||
// ---- Functions for changing the mailbox ----
|
||||
|
||||
async fn add_flags(&mut self, ident: UniqueIdent, flags: &[Flag]) -> Result<()> {
|
||||
let add_flag_op = self.uid_index.state().op_flag_add(ident, flags.to_vec());
|
||||
self.uid_index.push(add_flag_op).await
|
||||
}
|
||||
|
||||
async fn del_flags(&mut self, ident: UniqueIdent, flags: &[Flag]) -> Result<()> {
|
||||
let del_flag_op = self.uid_index.state().op_flag_del(ident, flags.to_vec());
|
||||
self.uid_index.push(del_flag_op).await
|
||||
}
|
||||
|
||||
async fn set_flags(&mut self, ident: UniqueIdent, flags: &[Flag]) -> Result<()> {
|
||||
let set_flag_op = self.uid_index.state().op_flag_set(ident, flags.to_vec());
|
||||
self.uid_index.push(set_flag_op).await
|
||||
}
|
||||
|
||||
async fn append(
|
||||
&mut self,
|
||||
mail: IMF<'_>,
|
||||
ident: Option<UniqueIdent>,
|
||||
flags: &[Flag],
|
||||
) -> Result<(ImapUidvalidity, ImapUid, ModSeq)> {
|
||||
let ident = ident.unwrap_or_else(gen_ident);
|
||||
let message_key = gen_key();
|
||||
|
||||
futures::try_join!(
|
||||
async {
|
||||
// Encrypt and save mail body
|
||||
let message_blob = cryptoblob::seal(mail.raw, &message_key)?;
|
||||
self.storage
|
||||
.blob_insert(BlobVal::new(
|
||||
BlobRef(format!("{}/{}", self.mail_path, ident)),
|
||||
message_blob,
|
||||
))
|
||||
.await?;
|
||||
Ok::<_, anyhow::Error>(())
|
||||
},
|
||||
async {
|
||||
// Save mail meta
|
||||
let meta = MailMeta {
|
||||
internaldate: now_msec(),
|
||||
headers: mail.parsed.raw_headers.to_vec(),
|
||||
message_key: message_key.clone(),
|
||||
rfc822_size: mail.raw.len(),
|
||||
};
|
||||
let meta_blob = seal_serialize(&meta, &self.encryption_key)?;
|
||||
self.storage
|
||||
.row_insert(vec![RowVal::new(
|
||||
RowRef::new(&self.mail_path, &ident.to_string()),
|
||||
meta_blob,
|
||||
)])
|
||||
.await?;
|
||||
Ok::<_, anyhow::Error>(())
|
||||
},
|
||||
self.uid_index.opportunistic_sync()
|
||||
)?;
|
||||
|
||||
// Add mail to Bayou mail index
|
||||
let uid_state = self.uid_index.state();
|
||||
let add_mail_op = uid_state.op_mail_add(ident, flags.to_vec());
|
||||
|
||||
let uidvalidity = uid_state.uidvalidity;
|
||||
let (uid, modseq) = match add_mail_op {
|
||||
UidIndexOp::MailAdd(_, uid, modseq, _) => (uid, modseq),
|
||||
_ => unreachable!(),
|
||||
};
|
||||
|
||||
self.uid_index.push(add_mail_op).await?;
|
||||
|
||||
Ok((uidvalidity, uid, modseq))
|
||||
}
|
||||
|
||||
async fn append_from_s3<'a>(
|
||||
&mut self,
|
||||
mail: IMF<'a>,
|
||||
ident: UniqueIdent,
|
||||
blob_src: storage::BlobRef,
|
||||
message_key: Key,
|
||||
) -> Result<()> {
|
||||
futures::try_join!(
|
||||
async {
|
||||
// Copy mail body from previous location
|
||||
let blob_dst = BlobRef(format!("{}/{}", self.mail_path, ident));
|
||||
self.storage.blob_copy(&blob_src, &blob_dst).await?;
|
||||
Ok::<_, anyhow::Error>(())
|
||||
},
|
||||
async {
|
||||
// Save mail meta
|
||||
let meta = MailMeta {
|
||||
internaldate: now_msec(),
|
||||
headers: mail.parsed.raw_headers.to_vec(),
|
||||
message_key: message_key.clone(),
|
||||
rfc822_size: mail.raw.len(),
|
||||
};
|
||||
let meta_blob = seal_serialize(&meta, &self.encryption_key)?;
|
||||
self.storage
|
||||
.row_insert(vec![RowVal::new(
|
||||
RowRef::new(&self.mail_path, &ident.to_string()),
|
||||
meta_blob,
|
||||
)])
|
||||
.await?;
|
||||
Ok::<_, anyhow::Error>(())
|
||||
},
|
||||
self.uid_index.opportunistic_sync()
|
||||
)?;
|
||||
|
||||
// Add mail to Bayou mail index
|
||||
let add_mail_op = self.uid_index.state().op_mail_add(ident, vec![]);
|
||||
self.uid_index.push(add_mail_op).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn delete(&mut self, ident: UniqueIdent) -> Result<()> {
|
||||
if !self.uid_index.state().table.contains_key(&ident) {
|
||||
bail!("Cannot delete mail that doesn't exit");
|
||||
}
|
||||
|
||||
let del_mail_op = self.uid_index.state().op_mail_del(ident);
|
||||
self.uid_index.push(del_mail_op).await?;
|
||||
|
||||
futures::try_join!(
|
||||
async {
|
||||
// Delete mail body from S3
|
||||
self.storage
|
||||
.blob_rm(&BlobRef(format!("{}/{}", self.mail_path, ident)))
|
||||
.await?;
|
||||
Ok::<_, anyhow::Error>(())
|
||||
},
|
||||
async {
|
||||
// Delete mail meta from K2V
|
||||
let sk = ident.to_string();
|
||||
let res = self
|
||||
.storage
|
||||
.row_fetch(&storage::Selector::Single(&RowRef::new(
|
||||
&self.mail_path,
|
||||
&sk,
|
||||
)))
|
||||
.await?;
|
||||
if let Some(row_val) = res.into_iter().next() {
|
||||
self.storage
|
||||
.row_rm(&storage::Selector::Single(&row_val.row_ref))
|
||||
.await?;
|
||||
}
|
||||
Ok::<_, anyhow::Error>(())
|
||||
}
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn copy_from(
|
||||
&mut self,
|
||||
from: &MailboxInternal,
|
||||
source_id: UniqueIdent,
|
||||
) -> Result<UniqueIdent> {
|
||||
let new_id = gen_ident();
|
||||
self.copy_internal(from, source_id, new_id).await?;
|
||||
Ok(new_id)
|
||||
}
|
||||
|
||||
async fn move_from(&mut self, from: &mut MailboxInternal, id: UniqueIdent) -> Result<()> {
|
||||
self.copy_internal(from, id, id).await?;
|
||||
from.delete(id).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn copy_internal(
|
||||
&mut self,
|
||||
from: &MailboxInternal,
|
||||
source_id: UniqueIdent,
|
||||
new_id: UniqueIdent,
|
||||
) -> Result<()> {
|
||||
if self.encryption_key != from.encryption_key {
|
||||
bail!("Message to be copied/moved does not belong to same account.");
|
||||
}
|
||||
|
||||
let flags = from
|
||||
.uid_index
|
||||
.state()
|
||||
.table
|
||||
.get(&source_id)
|
||||
.ok_or(anyhow!("Source mail not found"))?
|
||||
.2
|
||||
.clone();
|
||||
|
||||
futures::try_join!(
|
||||
async {
|
||||
let dst = BlobRef(format!("{}/{}", self.mail_path, new_id));
|
||||
let src = BlobRef(format!("{}/{}", from.mail_path, source_id));
|
||||
self.storage.blob_copy(&src, &dst).await?;
|
||||
Ok::<_, anyhow::Error>(())
|
||||
},
|
||||
async {
|
||||
// Copy mail meta in K2V
|
||||
let meta = &from.fetch_meta(&[source_id]).await?[0];
|
||||
let meta_blob = seal_serialize(meta, &self.encryption_key)?;
|
||||
self.storage
|
||||
.row_insert(vec![RowVal::new(
|
||||
RowRef::new(&self.mail_path, &new_id.to_string()),
|
||||
meta_blob,
|
||||
)])
|
||||
.await?;
|
||||
Ok::<_, anyhow::Error>(())
|
||||
},
|
||||
self.uid_index.opportunistic_sync(),
|
||||
)?;
|
||||
|
||||
// Add mail to Bayou mail index
|
||||
let add_mail_op = self.uid_index.state().op_mail_add(new_id, flags);
|
||||
self.uid_index.push(add_mail_op).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// Can be useful to debug so we want this code
|
||||
// to be available to developers
|
||||
#[allow(dead_code)]
|
||||
fn dump(uid_index: &Bayou<UidIndex>) {
|
||||
let s = uid_index.state();
|
||||
println!("---- MAILBOX STATE ----");
|
||||
println!("UIDVALIDITY {}", s.uidvalidity);
|
||||
println!("UIDNEXT {}", s.uidnext);
|
||||
println!("INTERNALSEQ {}", s.internalseq);
|
||||
for (uid, ident) in s.idx_by_uid.iter() {
|
||||
println!(
|
||||
"{} {} {}",
|
||||
uid,
|
||||
hex::encode(ident.0),
|
||||
s.table.get(ident).cloned().unwrap().2.join(", ")
|
||||
);
|
||||
}
|
||||
println!();
|
||||
}
|
||||
|
||||
// ----
|
||||
|
||||
/// The metadata of a message that is stored in K2V
|
||||
/// at pk = mail/<mailbox uuid>, sk = <message uuid>
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct MailMeta {
|
||||
/// INTERNALDATE field (milliseconds since epoch)
|
||||
pub internaldate: u64,
|
||||
/// Headers of the message
|
||||
pub headers: Vec<u8>,
|
||||
/// Secret key for decrypting entire message
|
||||
pub message_key: Key,
|
||||
/// RFC822 size
|
||||
pub rfc822_size: usize,
|
||||
}
|
||||
|
||||
impl MailMeta {
|
||||
fn try_merge(&mut self, other: Self) -> Result<()> {
|
||||
if self.headers != other.headers
|
||||
|| self.message_key != other.message_key
|
||||
|| self.rfc822_size != other.rfc822_size
|
||||
{
|
||||
bail!("Conflicting MailMeta values.");
|
||||
}
|
||||
self.internaldate = std::cmp::max(self.internaldate, other.internaldate);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
178
src/mail/mod.rs
178
src/mail/mod.rs
|
|
@ -1,27 +1,167 @@
|
|||
pub mod mail_ident;
|
||||
mod uidindex;
|
||||
|
||||
use std::convert::TryFrom;
|
||||
|
||||
pub mod incoming;
|
||||
pub mod mailbox;
|
||||
pub mod query;
|
||||
pub mod snapshot;
|
||||
pub mod uidindex;
|
||||
pub mod unique_ident;
|
||||
pub mod user;
|
||||
use anyhow::Result;
|
||||
use k2v_client::K2vClient;
|
||||
use rusoto_s3::S3Client;
|
||||
|
||||
use crate::bayou::Bayou;
|
||||
use crate::cryptoblob::Key;
|
||||
use crate::login::Credentials;
|
||||
use crate::mail::mail_ident::*;
|
||||
use crate::mail::uidindex::*;
|
||||
|
||||
// Internet Message Format
|
||||
// aka RFC 822 - RFC 2822 - RFC 5322
|
||||
// 2023-05-15 don't want to refactor this struct now.
|
||||
#[allow(clippy::upper_case_acronyms)]
|
||||
pub struct IMF<'a> {
|
||||
raw: &'a [u8],
|
||||
parsed: eml_codec::part::composite::Message<'a>,
|
||||
pub struct IMF(Vec<u8>);
|
||||
|
||||
pub struct Summary<'a> {
|
||||
pub validity: ImapUidvalidity,
|
||||
pub next: ImapUid,
|
||||
pub exists: u32,
|
||||
pub recent: u32,
|
||||
pub flags: FlagIter<'a>,
|
||||
pub unseen: Option<&'a ImapUid>,
|
||||
}
|
||||
|
||||
impl<'a> TryFrom<&'a [u8]> for IMF<'a> {
|
||||
type Error = ();
|
||||
|
||||
fn try_from(body: &'a [u8]) -> Result<IMF<'a>, ()> {
|
||||
let parsed = eml_codec::parse_message(body).or(Err(()))?.1;
|
||||
Ok(Self { raw: body, parsed })
|
||||
impl std::fmt::Display for Summary<'_> {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||
write!(
|
||||
f,
|
||||
"uidvalidity: {}, uidnext: {}, exists: {}",
|
||||
self.validity, self.next, self.exists
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Non standard but common flags:
|
||||
// https://www.iana.org/assignments/imap-jmap-keywords/imap-jmap-keywords.xhtml
|
||||
pub struct Mailbox {
|
||||
bucket: String,
|
||||
pub name: String,
|
||||
key: Key,
|
||||
|
||||
k2v: K2vClient,
|
||||
s3: S3Client,
|
||||
|
||||
uid_index: Bayou<UidIndex>,
|
||||
}
|
||||
|
||||
impl Mailbox {
|
||||
pub fn new(creds: &Credentials, name: String) -> Result<Self> {
|
||||
let uid_index = Bayou::<UidIndex>::new(creds, name.clone())?;
|
||||
|
||||
Ok(Self {
|
||||
bucket: creds.bucket().to_string(),
|
||||
name,
|
||||
key: creds.keys.master.clone(),
|
||||
k2v: creds.k2v_client()?,
|
||||
s3: creds.s3_client()?,
|
||||
uid_index,
|
||||
})
|
||||
}
|
||||
|
||||
// Get a summary of the mailbox, useful for the SELECT command for example
|
||||
pub async fn summary(&mut self) -> Result<Summary> {
|
||||
self.uid_index.sync().await?;
|
||||
let state = self.uid_index.state();
|
||||
|
||||
let unseen = state
|
||||
.idx_by_flag
|
||||
.get(&"$unseen".to_string())
|
||||
.and_then(|os| os.get_min());
|
||||
let recent = state
|
||||
.idx_by_flag
|
||||
.get(&"\\Recent".to_string())
|
||||
.map(|os| os.len())
|
||||
.unwrap_or(0);
|
||||
|
||||
return Ok(Summary {
|
||||
validity: state.uidvalidity,
|
||||
next: state.uidnext,
|
||||
exists: u32::try_from(state.idx_by_uid.len())?,
|
||||
recent: u32::try_from(recent)?,
|
||||
flags: state.idx_by_flag.flags(),
|
||||
unseen,
|
||||
});
|
||||
}
|
||||
|
||||
// Insert an email in the mailbox
|
||||
pub async fn append(&mut self, msg: IMF) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Copy an email from an external to this mailbox
|
||||
// @FIXME is it needed or could we implement it with append?
|
||||
pub async fn copy(&mut self, mailbox: String, uid: ImapUid) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Delete all emails with the \Delete flag in the mailbox
|
||||
// Can be called by CLOSE and EXPUNGE
|
||||
// @FIXME do we want to implement this feature or a simpler "delete" command
|
||||
// The controller could then "fetch \Delete" and call delete on each email?
|
||||
pub async fn expunge(&mut self) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Update flags of a range of emails
|
||||
pub async fn store(&mut self) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn fetch(&mut self) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn test(&mut self) -> Result<()> {
|
||||
self.uid_index.sync().await?;
|
||||
|
||||
dump(&self.uid_index);
|
||||
|
||||
let add_mail_op = self
|
||||
.uid_index
|
||||
.state()
|
||||
.op_mail_add(gen_ident(), vec!["\\Unseen".into()]);
|
||||
self.uid_index.push(add_mail_op).await?;
|
||||
|
||||
dump(&self.uid_index);
|
||||
|
||||
if self.uid_index.state().idx_by_uid.len() > 6 {
|
||||
for i in 0..2 {
|
||||
let (_, ident) = self
|
||||
.uid_index
|
||||
.state()
|
||||
.idx_by_uid
|
||||
.iter()
|
||||
.skip(3 + i)
|
||||
.next()
|
||||
.unwrap();
|
||||
let del_mail_op = self.uid_index.state().op_mail_del(*ident);
|
||||
self.uid_index.push(del_mail_op).await?;
|
||||
|
||||
dump(&self.uid_index);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn dump(uid_index: &Bayou<UidIndex>) {
|
||||
let s = uid_index.state();
|
||||
println!("---- MAILBOX STATE ----");
|
||||
println!("UIDVALIDITY {}", s.uidvalidity);
|
||||
println!("UIDNEXT {}", s.uidnext);
|
||||
println!("INTERNALSEQ {}", s.internalseq);
|
||||
for (uid, ident) in s.idx_by_uid.iter() {
|
||||
println!(
|
||||
"{} {} {}",
|
||||
uid,
|
||||
hex::encode(ident.0),
|
||||
s.table.get(ident).cloned().unwrap().1.join(", ")
|
||||
);
|
||||
}
|
||||
println!("");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,137 +0,0 @@
|
|||
use super::mailbox::MailMeta;
|
||||
use super::snapshot::FrozenMailbox;
|
||||
use super::unique_ident::UniqueIdent;
|
||||
use anyhow::Result;
|
||||
use futures::future::FutureExt;
|
||||
use futures::stream::{BoxStream, Stream, StreamExt};
|
||||
|
||||
/// Query is in charge of fetching efficiently
|
||||
/// requested data for a list of emails
|
||||
pub struct Query<'a, 'b> {
|
||||
pub frozen: &'a FrozenMailbox,
|
||||
pub emails: &'b [UniqueIdent],
|
||||
pub scope: QueryScope,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum QueryScope {
|
||||
Index,
|
||||
Partial,
|
||||
Full,
|
||||
}
|
||||
impl QueryScope {
|
||||
pub fn union(&self, other: &QueryScope) -> QueryScope {
|
||||
match (self, other) {
|
||||
(QueryScope::Full, _) | (_, QueryScope::Full) => QueryScope::Full,
|
||||
(QueryScope::Partial, _) | (_, QueryScope::Partial) => QueryScope::Partial,
|
||||
(QueryScope::Index, QueryScope::Index) => QueryScope::Index,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//type QueryResultStream = Box<dyn Stream<Item = Result<QueryResult>>>;
|
||||
|
||||
impl<'a, 'b> Query<'a, 'b> {
|
||||
pub fn fetch(&self) -> BoxStream<Result<QueryResult>> {
|
||||
match self.scope {
|
||||
QueryScope::Index => Box::pin(
|
||||
futures::stream::iter(self.emails)
|
||||
.map(|&uuid| Ok(QueryResult::IndexResult { uuid })),
|
||||
),
|
||||
QueryScope::Partial => Box::pin(self.partial()),
|
||||
QueryScope::Full => Box::pin(self.full()),
|
||||
}
|
||||
}
|
||||
|
||||
// --- functions below are private *for reasons*
|
||||
fn partial<'d>(&'d self) -> impl Stream<Item = Result<QueryResult>> + 'd + Send {
|
||||
async move {
|
||||
let maybe_meta_list: Result<Vec<MailMeta>> =
|
||||
self.frozen.mailbox.fetch_meta(self.emails).await;
|
||||
let list_res = maybe_meta_list
|
||||
.map(|meta_list| {
|
||||
meta_list
|
||||
.into_iter()
|
||||
.zip(self.emails)
|
||||
.map(|(metadata, &uuid)| Ok(QueryResult::PartialResult { uuid, metadata }))
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_else(|e| vec![Err(e)]);
|
||||
|
||||
futures::stream::iter(list_res)
|
||||
}
|
||||
.flatten_stream()
|
||||
}
|
||||
|
||||
fn full<'d>(&'d self) -> impl Stream<Item = Result<QueryResult>> + 'd + Send {
|
||||
self.partial().then(move |maybe_meta| async move {
|
||||
let meta = maybe_meta?;
|
||||
|
||||
let content = self
|
||||
.frozen
|
||||
.mailbox
|
||||
.fetch_full(
|
||||
*meta.uuid(),
|
||||
&meta
|
||||
.metadata()
|
||||
.expect("meta to be PartialResult")
|
||||
.message_key,
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(meta.into_full(content).expect("meta to be PartialResult"))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum QueryResult {
|
||||
IndexResult {
|
||||
uuid: UniqueIdent,
|
||||
},
|
||||
PartialResult {
|
||||
uuid: UniqueIdent,
|
||||
metadata: MailMeta,
|
||||
},
|
||||
FullResult {
|
||||
uuid: UniqueIdent,
|
||||
metadata: MailMeta,
|
||||
content: Vec<u8>,
|
||||
},
|
||||
}
|
||||
impl QueryResult {
|
||||
pub fn uuid(&self) -> &UniqueIdent {
|
||||
match self {
|
||||
Self::IndexResult { uuid, .. } => uuid,
|
||||
Self::PartialResult { uuid, .. } => uuid,
|
||||
Self::FullResult { uuid, .. } => uuid,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn metadata(&self) -> Option<&MailMeta> {
|
||||
match self {
|
||||
Self::IndexResult { .. } => None,
|
||||
Self::PartialResult { metadata, .. } => Some(metadata),
|
||||
Self::FullResult { metadata, .. } => Some(metadata),
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub fn content(&self) -> Option<&[u8]> {
|
||||
match self {
|
||||
Self::FullResult { content, .. } => Some(content),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn into_full(self, content: Vec<u8>) -> Option<Self> {
|
||||
match self {
|
||||
Self::PartialResult { uuid, metadata } => Some(Self::FullResult {
|
||||
uuid,
|
||||
metadata,
|
||||
content,
|
||||
}),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,60 +0,0 @@
|
|||
use std::sync::Arc;
|
||||
|
||||
use anyhow::Result;
|
||||
|
||||
use super::mailbox::Mailbox;
|
||||
use super::query::{Query, QueryScope};
|
||||
use super::uidindex::UidIndex;
|
||||
use super::unique_ident::UniqueIdent;
|
||||
|
||||
/// A Frozen Mailbox has a snapshot of the current mailbox
|
||||
/// state that is desynchronized with the real mailbox state.
|
||||
/// It's up to the user to choose when their snapshot must be updated
|
||||
/// to give useful information to their clients
|
||||
pub struct FrozenMailbox {
|
||||
pub mailbox: Arc<Mailbox>,
|
||||
pub snapshot: UidIndex,
|
||||
}
|
||||
|
||||
impl FrozenMailbox {
|
||||
/// Create a snapshot from a mailbox, the mailbox + the snapshot
|
||||
/// becomes the "Frozen Mailbox".
|
||||
pub async fn new(mailbox: Arc<Mailbox>) -> Self {
|
||||
let state = mailbox.current_uid_index().await;
|
||||
|
||||
Self {
|
||||
mailbox,
|
||||
snapshot: state,
|
||||
}
|
||||
}
|
||||
|
||||
/// Force the synchronization of the inner mailbox
|
||||
/// but do not update the local snapshot
|
||||
pub async fn sync(&self) -> Result<()> {
|
||||
self.mailbox.opportunistic_sync().await
|
||||
}
|
||||
|
||||
/// Peek snapshot without updating the frozen mailbox
|
||||
/// Can be useful if you want to plan some writes
|
||||
/// while sending a diff to the client later
|
||||
pub async fn peek(&self) -> UidIndex {
|
||||
self.mailbox.current_uid_index().await
|
||||
}
|
||||
|
||||
/// Update the FrozenMailbox local snapshot.
|
||||
/// Returns the old snapshot, so you can build a diff
|
||||
pub async fn update(&mut self) -> UidIndex {
|
||||
let old_snapshot = self.snapshot.clone();
|
||||
self.snapshot = self.mailbox.current_uid_index().await;
|
||||
|
||||
old_snapshot
|
||||
}
|
||||
|
||||
pub fn query<'a, 'b>(&'a self, uuids: &'b [UniqueIdent], scope: QueryScope) -> Query<'a, 'b> {
|
||||
Query {
|
||||
frozen: self,
|
||||
emails: uuids,
|
||||
scope,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,98 +1,78 @@
|
|||
use std::num::{NonZeroU32, NonZeroU64};
|
||||
use std::num::NonZeroU32;
|
||||
|
||||
use im::{HashMap, OrdMap, OrdSet};
|
||||
use serde::{Deserialize, Deserializer, Serialize, Serializer};
|
||||
use im::{HashMap, HashSet, OrdMap, OrdSet};
|
||||
use serde::{de::Error, Deserialize, Deserializer, Serialize, Serializer};
|
||||
|
||||
use crate::bayou::*;
|
||||
use crate::mail::unique_ident::UniqueIdent;
|
||||
use crate::mail::mail_ident::MailIdent;
|
||||
|
||||
pub type ModSeq = NonZeroU64;
|
||||
pub type ImapUid = NonZeroU32;
|
||||
pub type ImapUidvalidity = NonZeroU32;
|
||||
pub type Flag = String;
|
||||
pub type IndexEntry = (ImapUid, ModSeq, Vec<Flag>);
|
||||
|
||||
#[derive(Clone)]
|
||||
/// A UidIndex handles the mutable part of a mailbox
|
||||
/// It is built by running the event log on it
|
||||
/// Each applied log generates a new UidIndex by cloning the previous one
|
||||
/// and applying the event. This is why we use immutable datastructures:
|
||||
/// they are cheap to clone.
|
||||
#[derive(Clone)]
|
||||
pub struct UidIndex {
|
||||
// Source of trust
|
||||
pub table: OrdMap<UniqueIdent, IndexEntry>,
|
||||
pub table: OrdMap<MailIdent, (ImapUid, Vec<Flag>)>,
|
||||
|
||||
// Indexes optimized for queries
|
||||
pub idx_by_uid: OrdMap<ImapUid, UniqueIdent>,
|
||||
pub idx_by_modseq: OrdMap<ModSeq, UniqueIdent>,
|
||||
pub idx_by_uid: OrdMap<ImapUid, MailIdent>,
|
||||
pub idx_by_flag: FlagIndex,
|
||||
|
||||
// "Public" Counters
|
||||
// Counters
|
||||
pub uidvalidity: ImapUidvalidity,
|
||||
pub uidnext: ImapUid,
|
||||
pub highestmodseq: ModSeq,
|
||||
|
||||
// "Internal" Counters
|
||||
pub internalseq: ImapUid,
|
||||
pub internalmodseq: ModSeq,
|
||||
}
|
||||
|
||||
#[derive(Clone, Serialize, Deserialize, Debug)]
|
||||
pub enum UidIndexOp {
|
||||
MailAdd(UniqueIdent, ImapUid, ModSeq, Vec<Flag>),
|
||||
MailDel(UniqueIdent),
|
||||
FlagAdd(UniqueIdent, ModSeq, Vec<Flag>),
|
||||
FlagDel(UniqueIdent, ModSeq, Vec<Flag>),
|
||||
FlagSet(UniqueIdent, ModSeq, Vec<Flag>),
|
||||
BumpUidvalidity(u32),
|
||||
MailAdd(MailIdent, ImapUid, Vec<Flag>),
|
||||
MailDel(MailIdent),
|
||||
FlagAdd(MailIdent, Vec<Flag>),
|
||||
FlagDel(MailIdent, Vec<Flag>),
|
||||
}
|
||||
|
||||
impl UidIndex {
|
||||
#[must_use]
|
||||
pub fn op_mail_add(&self, ident: UniqueIdent, flags: Vec<Flag>) -> UidIndexOp {
|
||||
UidIndexOp::MailAdd(ident, self.internalseq, self.internalmodseq, flags)
|
||||
pub fn op_mail_add(&self, ident: MailIdent, flags: Vec<Flag>) -> UidIndexOp {
|
||||
UidIndexOp::MailAdd(ident, self.internalseq, flags)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn op_mail_del(&self, ident: UniqueIdent) -> UidIndexOp {
|
||||
pub fn op_mail_del(&self, ident: MailIdent) -> UidIndexOp {
|
||||
UidIndexOp::MailDel(ident)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn op_flag_add(&self, ident: UniqueIdent, flags: Vec<Flag>) -> UidIndexOp {
|
||||
UidIndexOp::FlagAdd(ident, self.internalmodseq, flags)
|
||||
pub fn op_flag_add(&self, ident: MailIdent, flags: Vec<Flag>) -> UidIndexOp {
|
||||
UidIndexOp::FlagAdd(ident, flags)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn op_flag_del(&self, ident: UniqueIdent, flags: Vec<Flag>) -> UidIndexOp {
|
||||
UidIndexOp::FlagDel(ident, self.internalmodseq, flags)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn op_flag_set(&self, ident: UniqueIdent, flags: Vec<Flag>) -> UidIndexOp {
|
||||
UidIndexOp::FlagSet(ident, self.internalmodseq, flags)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn op_bump_uidvalidity(&self, count: u32) -> UidIndexOp {
|
||||
UidIndexOp::BumpUidvalidity(count)
|
||||
pub fn op_flag_del(&self, ident: MailIdent, flags: Vec<Flag>) -> UidIndexOp {
|
||||
UidIndexOp::FlagDel(ident, flags)
|
||||
}
|
||||
|
||||
// INTERNAL functions to keep state consistent
|
||||
|
||||
fn reg_email(&mut self, ident: UniqueIdent, uid: ImapUid, modseq: ModSeq, flags: &[Flag]) {
|
||||
fn reg_email(&mut self, ident: MailIdent, uid: ImapUid, flags: &Vec<Flag>) {
|
||||
// Insert the email in our table
|
||||
self.table.insert(ident, (uid, modseq, flags.to_owned()));
|
||||
self.table.insert(ident, (uid, flags.clone()));
|
||||
|
||||
// Update the indexes/caches
|
||||
self.idx_by_uid.insert(uid, ident);
|
||||
self.idx_by_flag.insert(uid, flags);
|
||||
self.idx_by_modseq.insert(modseq, ident);
|
||||
}
|
||||
|
||||
fn unreg_email(&mut self, ident: &UniqueIdent) {
|
||||
fn unreg_email(&mut self, ident: &MailIdent) {
|
||||
// We do nothing if the mail does not exist
|
||||
let (uid, modseq, flags) = match self.table.get(ident) {
|
||||
let (uid, flags) = match self.table.get(ident) {
|
||||
Some(v) => v,
|
||||
None => return,
|
||||
};
|
||||
|
|
@ -100,7 +80,6 @@ impl UidIndex {
|
|||
// Delete all cache entries
|
||||
self.idx_by_uid.remove(uid);
|
||||
self.idx_by_flag.remove(*uid, flags);
|
||||
self.idx_by_modseq.remove(modseq);
|
||||
|
||||
// Remove from source of trust
|
||||
self.table.remove(ident);
|
||||
|
|
@ -111,17 +90,11 @@ impl Default for UidIndex {
|
|||
fn default() -> Self {
|
||||
Self {
|
||||
table: OrdMap::new(),
|
||||
|
||||
idx_by_uid: OrdMap::new(),
|
||||
idx_by_modseq: OrdMap::new(),
|
||||
idx_by_flag: FlagIndex::new(),
|
||||
|
||||
uidvalidity: NonZeroU32::new(1).unwrap(),
|
||||
uidnext: NonZeroU32::new(1).unwrap(),
|
||||
highestmodseq: NonZeroU64::new(1).unwrap(),
|
||||
|
||||
internalseq: NonZeroU32::new(1).unwrap(),
|
||||
internalmodseq: NonZeroU64::new(1).unwrap(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -132,23 +105,17 @@ impl BayouState for UidIndex {
|
|||
fn apply(&self, op: &UidIndexOp) -> Self {
|
||||
let mut new = self.clone();
|
||||
match op {
|
||||
UidIndexOp::MailAdd(ident, uid, modseq, flags) => {
|
||||
// Change UIDValidity if there is a UID conflict or a MODSEQ conflict
|
||||
// @FIXME Need to prove that summing work
|
||||
// The intuition: we increase the UIDValidity by the number of possible conflicts
|
||||
if *uid < new.internalseq || *modseq < new.internalmodseq {
|
||||
let bump_uid = new.internalseq.get() - uid.get();
|
||||
let bump_modseq = (new.internalmodseq.get() - modseq.get()) as u32;
|
||||
UidIndexOp::MailAdd(ident, uid, flags) => {
|
||||
// Change UIDValidity if there is a conflict
|
||||
if *uid < new.internalseq {
|
||||
new.uidvalidity =
|
||||
NonZeroU32::new(new.uidvalidity.get() + bump_uid + bump_modseq).unwrap();
|
||||
NonZeroU32::new(new.uidvalidity.get() + new.internalseq.get() - uid.get())
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
// Assign the real uid of the email
|
||||
let new_uid = new.internalseq;
|
||||
|
||||
// Assign the real modseq of the email and its new flags
|
||||
let new_modseq = new.internalmodseq;
|
||||
|
||||
// Delete the previous entry if any.
|
||||
// Our proof has no assumption on `ident` uniqueness,
|
||||
// so we must handle this case even it is very unlikely
|
||||
|
|
@ -157,14 +124,10 @@ impl BayouState for UidIndex {
|
|||
new.unreg_email(ident);
|
||||
|
||||
// We record our email and update ou caches
|
||||
new.reg_email(*ident, new_uid, new_modseq, flags);
|
||||
new.reg_email(*ident, new_uid, flags);
|
||||
|
||||
// Update counters
|
||||
new.highestmodseq = new.internalmodseq;
|
||||
|
||||
new.internalseq = NonZeroU32::new(new.internalseq.get() + 1).unwrap();
|
||||
new.internalmodseq = NonZeroU64::new(new.internalmodseq.get() + 1).unwrap();
|
||||
|
||||
new.uidnext = new.internalseq;
|
||||
}
|
||||
UidIndexOp::MailDel(ident) => {
|
||||
|
|
@ -174,16 +137,8 @@ impl BayouState for UidIndex {
|
|||
// We update the counter
|
||||
new.internalseq = NonZeroU32::new(new.internalseq.get() + 1).unwrap();
|
||||
}
|
||||
UidIndexOp::FlagAdd(ident, candidate_modseq, new_flags) => {
|
||||
if let Some((uid, email_modseq, existing_flags)) = new.table.get_mut(ident) {
|
||||
// Bump UIDValidity if required
|
||||
if *candidate_modseq < new.internalmodseq {
|
||||
let bump_modseq =
|
||||
(new.internalmodseq.get() - candidate_modseq.get()) as u32;
|
||||
new.uidvalidity =
|
||||
NonZeroU32::new(new.uidvalidity.get() + bump_modseq).unwrap();
|
||||
}
|
||||
|
||||
UidIndexOp::FlagAdd(ident, new_flags) => {
|
||||
if let Some((uid, existing_flags)) = new.table.get_mut(ident) {
|
||||
// Add flags to the source of trust and the cache
|
||||
let mut to_add: Vec<Flag> = new_flags
|
||||
.iter()
|
||||
|
|
@ -191,83 +146,22 @@ impl BayouState for UidIndex {
|
|||
.cloned()
|
||||
.collect();
|
||||
new.idx_by_flag.insert(*uid, &to_add);
|
||||
*email_modseq = new.internalmodseq;
|
||||
new.idx_by_modseq.insert(new.internalmodseq, *ident);
|
||||
existing_flags.append(&mut to_add);
|
||||
|
||||
// Update counters
|
||||
new.highestmodseq = new.internalmodseq;
|
||||
new.internalmodseq = NonZeroU64::new(new.internalmodseq.get() + 1).unwrap();
|
||||
}
|
||||
}
|
||||
UidIndexOp::FlagDel(ident, candidate_modseq, rm_flags) => {
|
||||
if let Some((uid, email_modseq, existing_flags)) = new.table.get_mut(ident) {
|
||||
// Bump UIDValidity if required
|
||||
if *candidate_modseq < new.internalmodseq {
|
||||
let bump_modseq =
|
||||
(new.internalmodseq.get() - candidate_modseq.get()) as u32;
|
||||
new.uidvalidity =
|
||||
NonZeroU32::new(new.uidvalidity.get() + bump_modseq).unwrap();
|
||||
}
|
||||
|
||||
UidIndexOp::FlagDel(ident, rm_flags) => {
|
||||
if let Some((uid, existing_flags)) = new.table.get_mut(ident) {
|
||||
// Remove flags from the source of trust and the cache
|
||||
existing_flags.retain(|x| !rm_flags.contains(x));
|
||||
new.idx_by_flag.remove(*uid, rm_flags);
|
||||
|
||||
// Register that email has been modified
|
||||
new.idx_by_modseq.insert(new.internalmodseq, *ident);
|
||||
*email_modseq = new.internalmodseq;
|
||||
|
||||
// Update counters
|
||||
new.highestmodseq = new.internalmodseq;
|
||||
new.internalmodseq = NonZeroU64::new(new.internalmodseq.get() + 1).unwrap();
|
||||
}
|
||||
}
|
||||
UidIndexOp::FlagSet(ident, candidate_modseq, new_flags) => {
|
||||
if let Some((uid, email_modseq, existing_flags)) = new.table.get_mut(ident) {
|
||||
// Bump UIDValidity if required
|
||||
if *candidate_modseq < new.internalmodseq {
|
||||
let bump_modseq =
|
||||
(new.internalmodseq.get() - candidate_modseq.get()) as u32;
|
||||
new.uidvalidity =
|
||||
NonZeroU32::new(new.uidvalidity.get() + bump_modseq).unwrap();
|
||||
}
|
||||
|
||||
// Remove flags from the source of trust and the cache
|
||||
let (keep_flags, rm_flags): (Vec<String>, Vec<String>) = existing_flags
|
||||
.iter()
|
||||
.cloned()
|
||||
.partition(|x| new_flags.contains(x));
|
||||
*existing_flags = keep_flags;
|
||||
let mut to_add: Vec<Flag> = new_flags
|
||||
.iter()
|
||||
.filter(|f| !existing_flags.contains(f))
|
||||
.cloned()
|
||||
.collect();
|
||||
existing_flags.append(&mut to_add);
|
||||
new.idx_by_flag.remove(*uid, &rm_flags);
|
||||
new.idx_by_flag.insert(*uid, &to_add);
|
||||
|
||||
// Register that email has been modified
|
||||
new.idx_by_modseq.insert(new.internalmodseq, *ident);
|
||||
*email_modseq = new.internalmodseq;
|
||||
|
||||
// Update counters
|
||||
new.highestmodseq = new.internalmodseq;
|
||||
new.internalmodseq = NonZeroU64::new(new.internalmodseq.get() + 1).unwrap();
|
||||
}
|
||||
}
|
||||
UidIndexOp::BumpUidvalidity(count) => {
|
||||
new.uidvalidity = ImapUidvalidity::new(new.uidvalidity.get() + *count)
|
||||
.unwrap_or(ImapUidvalidity::new(u32::MAX).unwrap());
|
||||
}
|
||||
}
|
||||
new
|
||||
}
|
||||
}
|
||||
|
||||
// ---- FlagIndex implementation ----
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct FlagIndex(HashMap<Flag, OrdSet<ImapUid>>);
|
||||
pub type FlagIter<'a> = im::hashmap::Keys<'a, Flag, OrdSet<ImapUid>>;
|
||||
|
|
@ -276,7 +170,7 @@ impl FlagIndex {
|
|||
fn new() -> Self {
|
||||
Self(HashMap::new())
|
||||
}
|
||||
fn insert(&mut self, uid: ImapUid, flags: &[Flag]) {
|
||||
fn insert(&mut self, uid: ImapUid, flags: &Vec<Flag>) {
|
||||
flags.iter().for_each(|flag| {
|
||||
self.0
|
||||
.entry(flag.clone())
|
||||
|
|
@ -284,15 +178,10 @@ impl FlagIndex {
|
|||
.insert(uid);
|
||||
});
|
||||
}
|
||||
fn remove(&mut self, uid: ImapUid, flags: &[Flag]) {
|
||||
for flag in flags.iter() {
|
||||
if let Some(set) = self.0.get_mut(flag) {
|
||||
set.remove(&uid);
|
||||
if set.is_empty() {
|
||||
self.0.remove(flag);
|
||||
}
|
||||
}
|
||||
}
|
||||
fn remove(&mut self, uid: ImapUid, flags: &Vec<Flag>) -> () {
|
||||
flags.iter().for_each(|flag| {
|
||||
self.0.get_mut(flag).and_then(|set| set.remove(&uid));
|
||||
});
|
||||
}
|
||||
|
||||
pub fn get(&self, f: &Flag) -> Option<&OrdSet<ImapUid>> {
|
||||
|
|
@ -308,14 +197,10 @@ impl FlagIndex {
|
|||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
struct UidIndexSerializedRepr {
|
||||
mails: Vec<(ImapUid, ModSeq, UniqueIdent, Vec<Flag>)>,
|
||||
|
||||
mails: Vec<(ImapUid, MailIdent, Vec<Flag>)>,
|
||||
uidvalidity: ImapUidvalidity,
|
||||
uidnext: ImapUid,
|
||||
highestmodseq: ModSeq,
|
||||
|
||||
internalseq: ImapUid,
|
||||
internalmodseq: ModSeq,
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for UidIndex {
|
||||
|
|
@ -327,22 +212,16 @@ impl<'de> Deserialize<'de> for UidIndex {
|
|||
|
||||
let mut uidindex = UidIndex {
|
||||
table: OrdMap::new(),
|
||||
|
||||
idx_by_uid: OrdMap::new(),
|
||||
idx_by_modseq: OrdMap::new(),
|
||||
idx_by_flag: FlagIndex::new(),
|
||||
|
||||
uidvalidity: val.uidvalidity,
|
||||
uidnext: val.uidnext,
|
||||
highestmodseq: val.highestmodseq,
|
||||
|
||||
internalseq: val.internalseq,
|
||||
internalmodseq: val.internalmodseq,
|
||||
};
|
||||
|
||||
val.mails
|
||||
.iter()
|
||||
.for_each(|(uid, modseq, uuid, flags)| uidindex.reg_email(*uuid, *uid, *modseq, flags));
|
||||
.for_each(|(u, i, f)| uidindex.reg_email(*i, *u, f));
|
||||
|
||||
Ok(uidindex)
|
||||
}
|
||||
|
|
@ -354,17 +233,15 @@ impl Serialize for UidIndex {
|
|||
S: Serializer,
|
||||
{
|
||||
let mut mails = vec![];
|
||||
for (ident, (uid, modseq, flags)) in self.table.iter() {
|
||||
mails.push((*uid, *modseq, *ident, flags.clone()));
|
||||
for (ident, (uid, flags)) in self.table.iter() {
|
||||
mails.push((*uid, *ident, flags.clone()));
|
||||
}
|
||||
|
||||
let val = UidIndexSerializedRepr {
|
||||
mails,
|
||||
uidvalidity: self.uidvalidity,
|
||||
uidnext: self.uidnext,
|
||||
highestmodseq: self.highestmodseq,
|
||||
internalseq: self.internalseq,
|
||||
internalmodseq: self.internalmodseq,
|
||||
};
|
||||
|
||||
val.serialize(serializer)
|
||||
|
|
@ -383,29 +260,28 @@ mod tests {
|
|||
|
||||
// Add message 1
|
||||
{
|
||||
let m = UniqueIdent([0x01; 24]);
|
||||
let m = MailIdent([0x01; 24]);
|
||||
let f = vec!["\\Recent".to_string(), "\\Archive".to_string()];
|
||||
let ev = state.op_mail_add(m, f);
|
||||
state = state.apply(&ev);
|
||||
|
||||
// Early checks
|
||||
assert_eq!(state.table.len(), 1);
|
||||
let (uid, modseq, flags) = state.table.get(&m).unwrap();
|
||||
assert_eq!(*uid, NonZeroU32::new(1).unwrap());
|
||||
assert_eq!(*modseq, NonZeroU64::new(1).unwrap());
|
||||
let (uid, flags) = state.table.get(&m).unwrap();
|
||||
assert_eq!(*uid, 1);
|
||||
assert_eq!(flags.len(), 2);
|
||||
let ident = state.idx_by_uid.get(&NonZeroU32::new(1).unwrap()).unwrap();
|
||||
let ident = state.idx_by_uid.get(&1).unwrap();
|
||||
assert_eq!(&m, ident);
|
||||
let recent = state.idx_by_flag.0.get("\\Recent").unwrap();
|
||||
assert_eq!(recent.len(), 1);
|
||||
assert_eq!(recent.iter().next().unwrap(), &NonZeroU32::new(1).unwrap());
|
||||
assert_eq!(state.uidnext, NonZeroU32::new(2).unwrap());
|
||||
assert_eq!(state.uidvalidity, NonZeroU32::new(1).unwrap());
|
||||
assert_eq!(recent.iter().next().unwrap(), &1);
|
||||
assert_eq!(state.uidnext, 2);
|
||||
assert_eq!(state.uidvalidity, 1);
|
||||
}
|
||||
|
||||
// Add message 2
|
||||
{
|
||||
let m = UniqueIdent([0x02; 24]);
|
||||
let m = MailIdent([0x02; 24]);
|
||||
let f = vec!["\\Seen".to_string(), "\\Archive".to_string()];
|
||||
let ev = state.op_mail_add(m, f);
|
||||
state = state.apply(&ev);
|
||||
|
|
@ -416,7 +292,7 @@ mod tests {
|
|||
|
||||
// Add flags to message 1
|
||||
{
|
||||
let m = UniqueIdent([0x01; 24]);
|
||||
let m = MailIdent([0x01; 24]);
|
||||
let f = vec!["Important".to_string(), "$cl_1".to_string()];
|
||||
let ev = state.op_flag_add(m, f);
|
||||
state = state.apply(&ev);
|
||||
|
|
@ -424,7 +300,7 @@ mod tests {
|
|||
|
||||
// Delete flags from message 1
|
||||
{
|
||||
let m = UniqueIdent([0x01; 24]);
|
||||
let m = MailIdent([0x01; 24]);
|
||||
let f = vec!["\\Recent".to_string()];
|
||||
let ev = state.op_flag_del(m, f);
|
||||
state = state.apply(&ev);
|
||||
|
|
@ -435,7 +311,7 @@ mod tests {
|
|||
|
||||
// Delete message 2
|
||||
{
|
||||
let m = UniqueIdent([0x02; 24]);
|
||||
let m = MailIdent([0x02; 24]);
|
||||
let ev = state.op_mail_del(m);
|
||||
state = state.apply(&ev);
|
||||
|
||||
|
|
@ -445,29 +321,24 @@ mod tests {
|
|||
|
||||
// Add a message 3 concurrent to message 1 (trigger a uid validity change)
|
||||
{
|
||||
let m = UniqueIdent([0x03; 24]);
|
||||
let m = MailIdent([0x03; 24]);
|
||||
let f = vec!["\\Archive".to_string(), "\\Recent".to_string()];
|
||||
let ev = UidIndexOp::MailAdd(
|
||||
m,
|
||||
NonZeroU32::new(1).unwrap(),
|
||||
NonZeroU64::new(1).unwrap(),
|
||||
f,
|
||||
);
|
||||
let ev = UidIndexOp::MailAdd(m, 1, f);
|
||||
state = state.apply(&ev);
|
||||
}
|
||||
|
||||
// Checks
|
||||
{
|
||||
assert_eq!(state.table.len(), 2);
|
||||
assert!(state.uidvalidity > NonZeroU32::new(1).unwrap());
|
||||
assert!(state.uidvalidity > 1);
|
||||
|
||||
let (last_uid, ident) = state.idx_by_uid.get_max().unwrap();
|
||||
assert_eq!(ident, &UniqueIdent([0x03; 24]));
|
||||
assert_eq!(ident, &MailIdent([0x03; 24]));
|
||||
|
||||
let archive = state.idx_by_flag.0.get("\\Archive").unwrap();
|
||||
assert_eq!(archive.len(), 2);
|
||||
let mut iter = archive.iter();
|
||||
assert_eq!(iter.next().unwrap(), &NonZeroU32::new(1).unwrap());
|
||||
assert_eq!(iter.next().unwrap(), &1);
|
||||
assert_eq!(iter.next().unwrap(), last_uid);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
500
src/mail/user.rs
500
src/mail/user.rs
|
|
@ -1,500 +0,0 @@
|
|||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::sync::{Arc, Weak};
|
||||
|
||||
use anyhow::{anyhow, bail, Result};
|
||||
use lazy_static::lazy_static;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::sync::watch;
|
||||
|
||||
use crate::cryptoblob::{open_deserialize, seal_serialize};
|
||||
use crate::login::Credentials;
|
||||
use crate::mail::incoming::incoming_mail_watch_process;
|
||||
use crate::mail::mailbox::Mailbox;
|
||||
use crate::mail::uidindex::ImapUidvalidity;
|
||||
use crate::mail::unique_ident::{gen_ident, UniqueIdent};
|
||||
use crate::storage;
|
||||
use crate::timestamp::now_msec;
|
||||
|
||||
pub const MAILBOX_HIERARCHY_DELIMITER: char = '.';
|
||||
|
||||
/// INBOX is the only mailbox that must always exist.
|
||||
/// It is created automatically when the account is created.
|
||||
/// IMAP allows the user to rename INBOX to something else,
|
||||
/// in this case all messages from INBOX are moved to a mailbox
|
||||
/// with the new name and the INBOX mailbox still exists and is empty.
|
||||
/// In our implementation, we indeed move the underlying mailbox
|
||||
/// to the new name (i.e. the new name has the same id as the previous
|
||||
/// INBOX), and we create a new empty mailbox for INBOX.
|
||||
pub const INBOX: &str = "INBOX";
|
||||
|
||||
/// For convenience purpose, we also create some special mailbox
|
||||
/// that are described in RFC6154 SPECIAL-USE
|
||||
/// @FIXME maybe it should be a configuration parameter
|
||||
/// @FIXME maybe we should have a per-mailbox flag mechanism, either an enum or a string, so we
|
||||
/// track which mailbox is used for what.
|
||||
/// @FIXME Junk could be useful but we don't have any antispam solution yet so...
|
||||
/// @FIXME IMAP supports virtual mailbox. \All or \Flagged are intended to be virtual mailboxes.
|
||||
/// \Trash might be one, or not one. I don't know what we should do there.
|
||||
pub const DRAFTS: &str = "Drafts";
|
||||
pub const ARCHIVE: &str = "Archive";
|
||||
pub const SENT: &str = "Sent";
|
||||
pub const TRASH: &str = "Trash";
|
||||
|
||||
const MAILBOX_LIST_PK: &str = "mailboxes";
|
||||
const MAILBOX_LIST_SK: &str = "list";
|
||||
|
||||
pub struct User {
|
||||
pub username: String,
|
||||
pub creds: Credentials,
|
||||
pub storage: storage::Store,
|
||||
pub mailboxes: std::sync::Mutex<HashMap<UniqueIdent, Weak<Mailbox>>>,
|
||||
|
||||
tx_inbox_id: watch::Sender<Option<(UniqueIdent, ImapUidvalidity)>>,
|
||||
}
|
||||
|
||||
impl User {
|
||||
pub async fn new(username: String, creds: Credentials) -> Result<Arc<Self>> {
|
||||
let cache_key = (username.clone(), creds.storage.unique());
|
||||
|
||||
{
|
||||
let cache = USER_CACHE.lock().unwrap();
|
||||
if let Some(u) = cache.get(&cache_key).and_then(Weak::upgrade) {
|
||||
return Ok(u);
|
||||
}
|
||||
}
|
||||
|
||||
let user = Self::open(username, creds).await?;
|
||||
|
||||
let mut cache = USER_CACHE.lock().unwrap();
|
||||
if let Some(concurrent_user) = cache.get(&cache_key).and_then(Weak::upgrade) {
|
||||
drop(user);
|
||||
Ok(concurrent_user)
|
||||
} else {
|
||||
cache.insert(cache_key, Arc::downgrade(&user));
|
||||
Ok(user)
|
||||
}
|
||||
}
|
||||
|
||||
/// Lists user's available mailboxes
|
||||
pub async fn list_mailboxes(&self) -> Result<Vec<String>> {
|
||||
let (list, _ct) = self.load_mailbox_list().await?;
|
||||
Ok(list.existing_mailbox_names())
|
||||
}
|
||||
|
||||
/// Opens an existing mailbox given its IMAP name.
|
||||
pub async fn open_mailbox(&self, name: &str) -> Result<Option<Arc<Mailbox>>> {
|
||||
let (mut list, ct) = self.load_mailbox_list().await?;
|
||||
|
||||
//@FIXME it could be a trace or an opentelemtry trace thing.
|
||||
// Be careful to not leak sensible data
|
||||
/*
|
||||
eprintln!("List of mailboxes:");
|
||||
for ent in list.0.iter() {
|
||||
eprintln!(" - {:?}", ent);
|
||||
}
|
||||
*/
|
||||
|
||||
if let Some((uidvalidity, Some(mbid))) = list.get_mailbox(name) {
|
||||
let mb = self.open_mailbox_by_id(mbid, uidvalidity).await?;
|
||||
let mb_uidvalidity = mb.current_uid_index().await.uidvalidity;
|
||||
if mb_uidvalidity > uidvalidity {
|
||||
list.update_uidvalidity(name, mb_uidvalidity);
|
||||
self.save_mailbox_list(&list, ct).await?;
|
||||
}
|
||||
Ok(Some(mb))
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
/// Check whether mailbox exists
|
||||
pub async fn has_mailbox(&self, name: &str) -> Result<bool> {
|
||||
let (list, _ct) = self.load_mailbox_list().await?;
|
||||
Ok(list.has_mailbox(name))
|
||||
}
|
||||
|
||||
/// Creates a new mailbox in the user's IMAP namespace.
|
||||
pub async fn create_mailbox(&self, name: &str) -> Result<()> {
|
||||
if name.ends_with(MAILBOX_HIERARCHY_DELIMITER) {
|
||||
bail!("Invalid mailbox name: {}", name);
|
||||
}
|
||||
|
||||
let (mut list, ct) = self.load_mailbox_list().await?;
|
||||
match list.create_mailbox(name) {
|
||||
CreatedMailbox::Created(_, _) => {
|
||||
self.save_mailbox_list(&list, ct).await?;
|
||||
Ok(())
|
||||
}
|
||||
CreatedMailbox::Existed(_, _) => Err(anyhow!("Mailbox {} already exists", name)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Deletes a mailbox in the user's IMAP namespace.
|
||||
pub async fn delete_mailbox(&self, name: &str) -> Result<()> {
|
||||
if name == INBOX {
|
||||
bail!("Cannot delete INBOX");
|
||||
}
|
||||
|
||||
let (mut list, ct) = self.load_mailbox_list().await?;
|
||||
if list.has_mailbox(name) {
|
||||
//@TODO: actually delete mailbox contents
|
||||
list.set_mailbox(name, None);
|
||||
self.save_mailbox_list(&list, ct).await?;
|
||||
Ok(())
|
||||
} else {
|
||||
bail!("Mailbox {} does not exist", name);
|
||||
}
|
||||
}
|
||||
|
||||
/// Renames a mailbox in the user's IMAP namespace.
|
||||
pub async fn rename_mailbox(&self, old_name: &str, new_name: &str) -> Result<()> {
|
||||
let (mut list, ct) = self.load_mailbox_list().await?;
|
||||
|
||||
if old_name.ends_with(MAILBOX_HIERARCHY_DELIMITER) {
|
||||
bail!("Invalid mailbox name: {}", old_name);
|
||||
}
|
||||
if new_name.ends_with(MAILBOX_HIERARCHY_DELIMITER) {
|
||||
bail!("Invalid mailbox name: {}", new_name);
|
||||
}
|
||||
|
||||
if old_name == INBOX {
|
||||
list.rename_mailbox(old_name, new_name)?;
|
||||
if !self.ensure_inbox_exists(&mut list, &ct).await? {
|
||||
self.save_mailbox_list(&list, ct).await?;
|
||||
}
|
||||
} else {
|
||||
let names = list.existing_mailbox_names();
|
||||
|
||||
let old_name_w_delim = format!("{}{}", old_name, MAILBOX_HIERARCHY_DELIMITER);
|
||||
let new_name_w_delim = format!("{}{}", new_name, MAILBOX_HIERARCHY_DELIMITER);
|
||||
|
||||
if names
|
||||
.iter()
|
||||
.any(|x| x == new_name || x.starts_with(&new_name_w_delim))
|
||||
{
|
||||
bail!("Mailbox {} already exists", new_name);
|
||||
}
|
||||
|
||||
for name in names.iter() {
|
||||
if name == old_name {
|
||||
list.rename_mailbox(name, new_name)?;
|
||||
} else if let Some(tail) = name.strip_prefix(&old_name_w_delim) {
|
||||
let nnew = format!("{}{}", new_name_w_delim, tail);
|
||||
list.rename_mailbox(name, &nnew)?;
|
||||
}
|
||||
}
|
||||
|
||||
self.save_mailbox_list(&list, ct).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---- Internal user & mailbox management ----
|
||||
|
||||
async fn open(username: String, creds: Credentials) -> Result<Arc<Self>> {
|
||||
let storage = creds.storage.build().await?;
|
||||
|
||||
let (tx_inbox_id, rx_inbox_id) = watch::channel(None);
|
||||
|
||||
let user = Arc::new(Self {
|
||||
username,
|
||||
creds: creds.clone(),
|
||||
storage,
|
||||
tx_inbox_id,
|
||||
mailboxes: std::sync::Mutex::new(HashMap::new()),
|
||||
});
|
||||
|
||||
// Ensure INBOX exists (done inside load_mailbox_list)
|
||||
user.load_mailbox_list().await?;
|
||||
|
||||
tokio::spawn(incoming_mail_watch_process(
|
||||
Arc::downgrade(&user),
|
||||
user.creds.clone(),
|
||||
rx_inbox_id,
|
||||
));
|
||||
|
||||
Ok(user)
|
||||
}
|
||||
|
||||
pub(super) async fn open_mailbox_by_id(
|
||||
&self,
|
||||
id: UniqueIdent,
|
||||
min_uidvalidity: ImapUidvalidity,
|
||||
) -> Result<Arc<Mailbox>> {
|
||||
{
|
||||
let cache = self.mailboxes.lock().unwrap();
|
||||
if let Some(mb) = cache.get(&id).and_then(Weak::upgrade) {
|
||||
return Ok(mb);
|
||||
}
|
||||
}
|
||||
|
||||
let mb = Arc::new(Mailbox::open(&self.creds, id, min_uidvalidity).await?);
|
||||
|
||||
let mut cache = self.mailboxes.lock().unwrap();
|
||||
if let Some(concurrent_mb) = cache.get(&id).and_then(Weak::upgrade) {
|
||||
drop(mb); // we worked for nothing but at least we didn't starve someone else
|
||||
Ok(concurrent_mb)
|
||||
} else {
|
||||
cache.insert(id, Arc::downgrade(&mb));
|
||||
Ok(mb)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Mailbox list management ----
|
||||
|
||||
async fn load_mailbox_list(&self) -> Result<(MailboxList, Option<storage::RowRef>)> {
|
||||
let row_ref = storage::RowRef::new(MAILBOX_LIST_PK, MAILBOX_LIST_SK);
|
||||
let (mut list, row) = match self
|
||||
.storage
|
||||
.row_fetch(&storage::Selector::Single(&row_ref))
|
||||
.await
|
||||
{
|
||||
Err(storage::StorageError::NotFound) => (MailboxList::new(), None),
|
||||
Err(e) => return Err(e.into()),
|
||||
Ok(rv) => {
|
||||
let mut list = MailboxList::new();
|
||||
let (row_ref, row_vals) = match rv.into_iter().next() {
|
||||
Some(row_val) => (row_val.row_ref, row_val.value),
|
||||
None => (row_ref, vec![]),
|
||||
};
|
||||
|
||||
for v in row_vals {
|
||||
if let storage::Alternative::Value(vbytes) = v {
|
||||
let list2 =
|
||||
open_deserialize::<MailboxList>(&vbytes, &self.creds.keys.master)?;
|
||||
list.merge(list2);
|
||||
}
|
||||
}
|
||||
(list, Some(row_ref))
|
||||
}
|
||||
};
|
||||
|
||||
let is_default_mbx_missing = [DRAFTS, ARCHIVE, SENT, TRASH]
|
||||
.iter()
|
||||
.map(|mbx| list.create_mailbox(mbx))
|
||||
.fold(false, |acc, r| {
|
||||
acc || matches!(r, CreatedMailbox::Created(..))
|
||||
});
|
||||
let is_inbox_missing = self.ensure_inbox_exists(&mut list, &row).await?;
|
||||
if is_default_mbx_missing && !is_inbox_missing {
|
||||
// It's the only case where we created some mailboxes and not saved them
|
||||
// So we save them!
|
||||
self.save_mailbox_list(&list, row.clone()).await?;
|
||||
}
|
||||
|
||||
Ok((list, row))
|
||||
}
|
||||
|
||||
async fn ensure_inbox_exists(
|
||||
&self,
|
||||
list: &mut MailboxList,
|
||||
ct: &Option<storage::RowRef>,
|
||||
) -> Result<bool> {
|
||||
// If INBOX doesn't exist, create a new mailbox with that name
|
||||
// and save new mailbox list.
|
||||
// Also, ensure that the mpsc::watch that keeps track of the
|
||||
// inbox id is up-to-date.
|
||||
let saved;
|
||||
let (inbox_id, inbox_uidvalidity) = match list.create_mailbox(INBOX) {
|
||||
CreatedMailbox::Created(i, v) => {
|
||||
self.save_mailbox_list(list, ct.clone()).await?;
|
||||
saved = true;
|
||||
(i, v)
|
||||
}
|
||||
CreatedMailbox::Existed(i, v) => {
|
||||
saved = false;
|
||||
(i, v)
|
||||
}
|
||||
};
|
||||
let inbox_id = Some((inbox_id, inbox_uidvalidity));
|
||||
if *self.tx_inbox_id.borrow() != inbox_id {
|
||||
self.tx_inbox_id.send(inbox_id).unwrap();
|
||||
}
|
||||
|
||||
Ok(saved)
|
||||
}
|
||||
|
||||
async fn save_mailbox_list(
|
||||
&self,
|
||||
list: &MailboxList,
|
||||
ct: Option<storage::RowRef>,
|
||||
) -> Result<()> {
|
||||
let list_blob = seal_serialize(list, &self.creds.keys.master)?;
|
||||
let rref = ct.unwrap_or(storage::RowRef::new(MAILBOX_LIST_PK, MAILBOX_LIST_SK));
|
||||
let row_val = storage::RowVal::new(rref, list_blob);
|
||||
self.storage.row_insert(vec![row_val]).await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// ---- User's mailbox list (serialized in K2V) ----
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
struct MailboxList(BTreeMap<String, MailboxListEntry>);
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone, Copy, Debug)]
|
||||
struct MailboxListEntry {
|
||||
id_lww: (u64, Option<UniqueIdent>),
|
||||
uidvalidity: ImapUidvalidity,
|
||||
}
|
||||
|
||||
impl MailboxListEntry {
|
||||
fn merge(&mut self, other: &Self) {
|
||||
// Simple CRDT merge rule
|
||||
if other.id_lww.0 > self.id_lww.0
|
||||
|| (other.id_lww.0 == self.id_lww.0 && other.id_lww.1 > self.id_lww.1)
|
||||
{
|
||||
self.id_lww = other.id_lww;
|
||||
}
|
||||
self.uidvalidity = std::cmp::max(self.uidvalidity, other.uidvalidity);
|
||||
}
|
||||
}
|
||||
|
||||
impl MailboxList {
|
||||
fn new() -> Self {
|
||||
Self(BTreeMap::new())
|
||||
}
|
||||
|
||||
fn merge(&mut self, list2: Self) {
|
||||
for (k, v) in list2.0.into_iter() {
|
||||
if let Some(e) = self.0.get_mut(&k) {
|
||||
e.merge(&v);
|
||||
} else {
|
||||
self.0.insert(k, v);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn existing_mailbox_names(&self) -> Vec<String> {
|
||||
self.0
|
||||
.iter()
|
||||
.filter(|(_, v)| v.id_lww.1.is_some())
|
||||
.map(|(k, _)| k.to_string())
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn has_mailbox(&self, name: &str) -> bool {
|
||||
matches!(
|
||||
self.0.get(name),
|
||||
Some(MailboxListEntry {
|
||||
id_lww: (_, Some(_)),
|
||||
..
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
fn get_mailbox(&self, name: &str) -> Option<(ImapUidvalidity, Option<UniqueIdent>)> {
|
||||
self.0.get(name).map(
|
||||
|MailboxListEntry {
|
||||
id_lww: (_, mailbox_id),
|
||||
uidvalidity,
|
||||
}| (*uidvalidity, *mailbox_id),
|
||||
)
|
||||
}
|
||||
|
||||
/// Ensures mailbox `name` maps to id `id`.
|
||||
/// If it already mapped to that, returns None.
|
||||
/// If a change had to be done, returns Some(new uidvalidity in mailbox).
|
||||
fn set_mailbox(&mut self, name: &str, id: Option<UniqueIdent>) -> Option<ImapUidvalidity> {
|
||||
let (ts, id, uidvalidity) = match self.0.get_mut(name) {
|
||||
None => {
|
||||
if id.is_none() {
|
||||
return None;
|
||||
} else {
|
||||
(now_msec(), id, ImapUidvalidity::new(1).unwrap())
|
||||
}
|
||||
}
|
||||
Some(MailboxListEntry {
|
||||
id_lww,
|
||||
uidvalidity,
|
||||
}) => {
|
||||
if id_lww.1 == id {
|
||||
return None;
|
||||
} else {
|
||||
(
|
||||
std::cmp::max(id_lww.0 + 1, now_msec()),
|
||||
id,
|
||||
ImapUidvalidity::new(uidvalidity.get() + 1).unwrap(),
|
||||
)
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
self.0.insert(
|
||||
name.into(),
|
||||
MailboxListEntry {
|
||||
id_lww: (ts, id),
|
||||
uidvalidity,
|
||||
},
|
||||
);
|
||||
Some(uidvalidity)
|
||||
}
|
||||
|
||||
fn update_uidvalidity(&mut self, name: &str, new_uidvalidity: ImapUidvalidity) {
|
||||
match self.0.get_mut(name) {
|
||||
None => {
|
||||
self.0.insert(
|
||||
name.into(),
|
||||
MailboxListEntry {
|
||||
id_lww: (now_msec(), None),
|
||||
uidvalidity: new_uidvalidity,
|
||||
},
|
||||
);
|
||||
}
|
||||
Some(MailboxListEntry { uidvalidity, .. }) => {
|
||||
*uidvalidity = std::cmp::max(*uidvalidity, new_uidvalidity);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn create_mailbox(&mut self, name: &str) -> CreatedMailbox {
|
||||
if let Some(MailboxListEntry {
|
||||
id_lww: (_, Some(id)),
|
||||
uidvalidity,
|
||||
}) = self.0.get(name)
|
||||
{
|
||||
return CreatedMailbox::Existed(*id, *uidvalidity);
|
||||
}
|
||||
|
||||
let id = gen_ident();
|
||||
let uidvalidity = self.set_mailbox(name, Some(id)).unwrap();
|
||||
CreatedMailbox::Created(id, uidvalidity)
|
||||
}
|
||||
|
||||
fn rename_mailbox(&mut self, old_name: &str, new_name: &str) -> Result<()> {
|
||||
if let Some((uidvalidity, Some(mbid))) = self.get_mailbox(old_name) {
|
||||
if self.has_mailbox(new_name) {
|
||||
bail!(
|
||||
"Cannot rename {} into {}: {} already exists",
|
||||
old_name,
|
||||
new_name,
|
||||
new_name
|
||||
);
|
||||
}
|
||||
|
||||
self.set_mailbox(old_name, None);
|
||||
self.set_mailbox(new_name, Some(mbid));
|
||||
self.update_uidvalidity(new_name, uidvalidity);
|
||||
Ok(())
|
||||
} else {
|
||||
bail!(
|
||||
"Cannot rename {} into {}: {} doesn't exist",
|
||||
old_name,
|
||||
new_name,
|
||||
old_name
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enum CreatedMailbox {
|
||||
Created(UniqueIdent, ImapUidvalidity),
|
||||
Existed(UniqueIdent, ImapUidvalidity),
|
||||
}
|
||||
|
||||
// ---- User cache ----
|
||||
|
||||
lazy_static! {
|
||||
static ref USER_CACHE: std::sync::Mutex<HashMap<(String, storage::UnicityBuffer), Weak<User>>> =
|
||||
std::sync::Mutex::new(HashMap::new());
|
||||
}
|
||||
610
src/main.rs
610
src/main.rs
|
|
@ -1,26 +1,23 @@
|
|||
#![feature(async_fn_in_trait)]
|
||||
|
||||
mod auth;
|
||||
mod bayou;
|
||||
mod config;
|
||||
mod cryptoblob;
|
||||
mod imap;
|
||||
mod k2v_util;
|
||||
mod lmtp;
|
||||
mod login;
|
||||
mod mail;
|
||||
mod server;
|
||||
mod storage;
|
||||
mod timestamp;
|
||||
mod time;
|
||||
|
||||
use std::io::Read;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use anyhow::{bail, Context, Result};
|
||||
use anyhow::{bail, Result};
|
||||
use clap::{Parser, Subcommand};
|
||||
use nix::{sys::signal, unistd::Pid};
|
||||
use rand::prelude::*;
|
||||
|
||||
use rusoto_signature::Region;
|
||||
|
||||
use config::*;
|
||||
use cryptoblob::*;
|
||||
use login::{static_provider::*, *};
|
||||
use server::Server;
|
||||
|
||||
|
|
@ -29,391 +26,288 @@ use server::Server;
|
|||
struct Args {
|
||||
#[clap(subcommand)]
|
||||
command: Command,
|
||||
|
||||
/// A special mode dedicated to developers, NOT INTENDED FOR PRODUCTION
|
||||
#[clap(long)]
|
||||
dev: bool,
|
||||
|
||||
#[clap(
|
||||
short,
|
||||
long,
|
||||
env = "AEROGRAMME_CONFIG",
|
||||
default_value = "aerogramme.toml"
|
||||
)]
|
||||
/// Path to the main Aerogramme configuration file
|
||||
config_file: PathBuf,
|
||||
}
|
||||
|
||||
#[derive(Subcommand, Debug)]
|
||||
enum Command {
|
||||
#[clap(subcommand)]
|
||||
/// A daemon to be run by the end user, on a personal device
|
||||
Companion(CompanionCommand),
|
||||
|
||||
#[clap(subcommand)]
|
||||
/// A daemon to be run by the service provider, on a server
|
||||
Provider(ProviderCommand),
|
||||
|
||||
#[clap(subcommand)]
|
||||
/// Specific tooling, should not be part of a normal workflow, for debug & experimentation only
|
||||
Tools(ToolsCommand),
|
||||
//Test,
|
||||
}
|
||||
|
||||
#[derive(Subcommand, Debug)]
|
||||
enum ToolsCommand {
|
||||
/// Manage crypto roots
|
||||
#[clap(subcommand)]
|
||||
CryptoRoot(CryptoRootCommand),
|
||||
|
||||
PasswordHash {
|
||||
#[clap(env = "AEROGRAMME_PASSWORD")]
|
||||
maybe_password: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Subcommand, Debug)]
|
||||
enum CryptoRootCommand {
|
||||
/// Generate a new crypto-root protected with a password
|
||||
New {
|
||||
#[clap(env = "AEROGRAMME_PASSWORD")]
|
||||
maybe_password: Option<String>,
|
||||
},
|
||||
/// Generate a new clear text crypto-root, store it securely!
|
||||
NewClearText,
|
||||
/// Change the password of a crypto key
|
||||
ChangePassword {
|
||||
#[clap(env = "AEROGRAMME_OLD_PASSWORD")]
|
||||
maybe_old_password: Option<String>,
|
||||
|
||||
#[clap(env = "AEROGRAMME_NEW_PASSWORD")]
|
||||
maybe_new_password: Option<String>,
|
||||
|
||||
#[clap(short, long, env = "AEROGRAMME_CRYPTO_ROOT")]
|
||||
crypto_root: String,
|
||||
},
|
||||
/// From a given crypto-key, derive one containing only the public key
|
||||
DeriveIncoming {
|
||||
#[clap(short, long, env = "AEROGRAMME_CRYPTO_ROOT")]
|
||||
crypto_root: String,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Subcommand, Debug)]
|
||||
enum CompanionCommand {
|
||||
/// Runs the IMAP proxy
|
||||
Daemon,
|
||||
Reload {
|
||||
#[clap(short, long, env = "AEROGRAMME_PID")]
|
||||
pid: Option<i32>,
|
||||
},
|
||||
Wizard,
|
||||
#[clap(subcommand)]
|
||||
Account(AccountManagement),
|
||||
}
|
||||
|
||||
#[derive(Subcommand, Debug)]
|
||||
enum ProviderCommand {
|
||||
/// Runs the IMAP+LMTP server daemon
|
||||
Daemon,
|
||||
/// Reload the daemon
|
||||
Reload {
|
||||
#[clap(short, long, env = "AEROGRAMME_PID")]
|
||||
pid: Option<i32>,
|
||||
Server {
|
||||
#[clap(short, long, env = "CONFIG_FILE", default_value = "mailrage.toml")]
|
||||
config_file: PathBuf,
|
||||
},
|
||||
/// Manage static accounts
|
||||
#[clap(subcommand)]
|
||||
Account(AccountManagement),
|
||||
}
|
||||
|
||||
#[derive(Subcommand, Debug)]
|
||||
enum AccountManagement {
|
||||
/// Add an account
|
||||
Add {
|
||||
#[clap(short, long)]
|
||||
login: String,
|
||||
#[clap(short, long)]
|
||||
setup: PathBuf,
|
||||
/// TEST TEST TEST
|
||||
Test {
|
||||
#[clap(short, long, env = "CONFIG_FILE", default_value = "mailrage.toml")]
|
||||
config_file: PathBuf,
|
||||
},
|
||||
/// Delete an account
|
||||
Delete {
|
||||
#[clap(short, long)]
|
||||
login: String,
|
||||
/// Initializes key pairs for a user and adds a key decryption password
|
||||
FirstLogin {
|
||||
#[clap(flatten)]
|
||||
creds: StorageCredsArgs,
|
||||
#[clap(flatten)]
|
||||
user_secrets: UserSecretsArgs,
|
||||
},
|
||||
/// Change password for a given account
|
||||
ChangePassword {
|
||||
#[clap(env = "AEROGRAMME_OLD_PASSWORD")]
|
||||
maybe_old_password: Option<String>,
|
||||
|
||||
#[clap(env = "AEROGRAMME_NEW_PASSWORD")]
|
||||
maybe_new_password: Option<String>,
|
||||
|
||||
/// Initializes key pairs for a user and dumps keys to stdout for usage with static
|
||||
/// login provider
|
||||
InitializeLocalKeys {
|
||||
#[clap(flatten)]
|
||||
creds: StorageCredsArgs,
|
||||
},
|
||||
/// Adds a key decryption password for a user
|
||||
AddPassword {
|
||||
#[clap(flatten)]
|
||||
creds: StorageCredsArgs,
|
||||
#[clap(flatten)]
|
||||
user_secrets: UserSecretsArgs,
|
||||
/// Automatically generate password
|
||||
#[clap(short, long)]
|
||||
login: String,
|
||||
gen: bool,
|
||||
},
|
||||
/// Deletes a key decription password for a user
|
||||
DeletePassword {
|
||||
#[clap(flatten)]
|
||||
creds: StorageCredsArgs,
|
||||
#[clap(flatten)]
|
||||
user_secrets: UserSecretsArgs,
|
||||
/// Allow to delete all passwords
|
||||
#[clap(long)]
|
||||
allow_delete_all: bool,
|
||||
},
|
||||
/// Dumps all encryption keys for user
|
||||
ShowKeys {
|
||||
#[clap(flatten)]
|
||||
creds: StorageCredsArgs,
|
||||
#[clap(flatten)]
|
||||
user_secrets: UserSecretsArgs,
|
||||
},
|
||||
}
|
||||
|
||||
#[cfg(tokio_unstable)]
|
||||
fn tracer() {
|
||||
console_subscriber::init();
|
||||
#[derive(Parser, Debug)]
|
||||
struct StorageCredsArgs {
|
||||
/// Name of the region to use
|
||||
#[clap(short = 'r', long, env = "AWS_REGION")]
|
||||
region: String,
|
||||
/// Url of the endpoint to connect to for K2V
|
||||
#[clap(short = 'k', long, env = "K2V_ENDPOINT")]
|
||||
k2v_endpoint: String,
|
||||
/// Url of the endpoint to connect to for S3
|
||||
#[clap(short = 's', long, env = "S3_ENDPOINT")]
|
||||
s3_endpoint: String,
|
||||
/// Access key ID
|
||||
#[clap(short = 'A', long, env = "AWS_ACCESS_KEY_ID")]
|
||||
aws_access_key_id: String,
|
||||
/// Access key ID
|
||||
#[clap(short = 'S', long, env = "AWS_SECRET_ACCESS_KEY")]
|
||||
aws_secret_access_key: String,
|
||||
/// Bucket name
|
||||
#[clap(short = 'b', long, env = "BUCKET")]
|
||||
bucket: String,
|
||||
}
|
||||
|
||||
#[cfg(not(tokio_unstable))]
|
||||
fn tracer() {
|
||||
tracing_subscriber::fmt::init();
|
||||
#[derive(Parser, Debug)]
|
||||
struct UserSecretsArgs {
|
||||
/// User secret
|
||||
#[clap(short = 'U', long, env = "USER_SECRET")]
|
||||
user_secret: String,
|
||||
/// Alternate user secrets (comma-separated list of strings)
|
||||
#[clap(long, env = "ALTERNATE_USER_SECRETS", default_value = "")]
|
||||
alternate_user_secrets: String,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
if std::env::var("RUST_LOG").is_err() {
|
||||
std::env::set_var("RUST_LOG", "main=info,aerogramme=info,k2v_client=info")
|
||||
std::env::set_var("RUST_LOG", "main=info,mailrage=info,k2v_client=info")
|
||||
}
|
||||
|
||||
// Abort on panic (same behavior as in Go)
|
||||
std::panic::set_hook(Box::new(|panic_info| {
|
||||
eprintln!("{}", panic_info);
|
||||
eprintln!("{:?}", backtrace::Backtrace::new());
|
||||
std::process::abort();
|
||||
}));
|
||||
|
||||
tracer();
|
||||
tracing_subscriber::fmt::init();
|
||||
|
||||
let args = Args::parse();
|
||||
let any_config = if args.dev {
|
||||
use std::net::*;
|
||||
AnyConfig::Provider(ProviderConfig {
|
||||
pid: None,
|
||||
imap: None,
|
||||
imap_unsecure: Some(ImapUnsecureConfig {
|
||||
bind_addr: SocketAddr::new(IpAddr::V6(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1)), 1143),
|
||||
}),
|
||||
lmtp: Some(LmtpConfig {
|
||||
bind_addr: SocketAddr::new(IpAddr::V6(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1)), 1025),
|
||||
hostname: "example.tld".to_string(),
|
||||
}),
|
||||
auth: Some(AuthConfig {
|
||||
bind_addr: SocketAddr::new(
|
||||
IpAddr::V6(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1)),
|
||||
12345,
|
||||
),
|
||||
}),
|
||||
users: UserManagement::Demo,
|
||||
})
|
||||
} else {
|
||||
read_config(args.config_file)?
|
||||
};
|
||||
|
||||
match (&args.command, any_config) {
|
||||
(Command::Companion(subcommand), AnyConfig::Companion(config)) => match subcommand {
|
||||
CompanionCommand::Daemon => {
|
||||
let server = Server::from_companion_config(config).await?;
|
||||
server.run().await?;
|
||||
}
|
||||
CompanionCommand::Reload { pid } => reload(*pid, config.pid)?,
|
||||
CompanionCommand::Wizard => {
|
||||
unimplemented!();
|
||||
}
|
||||
CompanionCommand::Account(cmd) => {
|
||||
let user_file = config.users.user_list;
|
||||
account_management(&args.command, cmd, user_file)?;
|
||||
}
|
||||
},
|
||||
(Command::Provider(subcommand), AnyConfig::Provider(config)) => match subcommand {
|
||||
ProviderCommand::Daemon => {
|
||||
let server = Server::from_provider_config(config).await?;
|
||||
server.run().await?;
|
||||
}
|
||||
ProviderCommand::Reload { pid } => reload(*pid, config.pid)?,
|
||||
ProviderCommand::Account(cmd) => {
|
||||
let user_file = match config.users {
|
||||
UserManagement::Static(conf) => conf.user_list,
|
||||
_ => {
|
||||
panic!("Only static account management is supported from Aerogramme.")
|
||||
}
|
||||
};
|
||||
account_management(&args.command, cmd, user_file)?;
|
||||
}
|
||||
},
|
||||
(Command::Provider(_), AnyConfig::Companion(_)) => {
|
||||
bail!("Your want to run a 'Provider' command but your configuration file has role 'Companion'.");
|
||||
match args.command {
|
||||
Command::Server { config_file } => {
|
||||
let config = read_config(config_file)?;
|
||||
|
||||
let server = Server::new(config).await?;
|
||||
server.run().await?;
|
||||
}
|
||||
(Command::Companion(_), AnyConfig::Provider(_)) => {
|
||||
bail!("Your want to run a 'Companion' command but your configuration file has role 'Provider'.");
|
||||
Command::Test { config_file } => {
|
||||
let config = read_config(config_file)?;
|
||||
|
||||
let server = Server::new(config).await?;
|
||||
//server.test().await?;
|
||||
}
|
||||
(Command::Tools(subcommand), _) => match subcommand {
|
||||
ToolsCommand::PasswordHash { maybe_password } => {
|
||||
let password = match maybe_password {
|
||||
Some(pwd) => pwd.clone(),
|
||||
None => rpassword::prompt_password("Enter password: ")?,
|
||||
};
|
||||
println!("{}", hash_password(&password)?);
|
||||
Command::FirstLogin {
|
||||
creds,
|
||||
user_secrets,
|
||||
} => {
|
||||
let creds = make_storage_creds(creds);
|
||||
let user_secrets = make_user_secrets(user_secrets);
|
||||
|
||||
println!("Please enter your password for key decryption.");
|
||||
println!("If you are using LDAP login, this must be your LDAP password.");
|
||||
println!("If you are using the static login provider, enter any password, and this will also become your password for local IMAP access.");
|
||||
let password = rpassword::prompt_password("Enter password: ")?;
|
||||
let password_confirm = rpassword::prompt_password("Confirm password: ")?;
|
||||
if password != password_confirm {
|
||||
bail!("Passwords don't match.");
|
||||
}
|
||||
ToolsCommand::CryptoRoot(crcommand) => match crcommand {
|
||||
CryptoRootCommand::New { maybe_password } => {
|
||||
let password = match maybe_password {
|
||||
Some(pwd) => pwd.clone(),
|
||||
None => {
|
||||
let password = rpassword::prompt_password("Enter password: ")?;
|
||||
let password_confirm =
|
||||
rpassword::prompt_password("Confirm password: ")?;
|
||||
if password != password_confirm {
|
||||
bail!("Passwords don't match.");
|
||||
}
|
||||
password
|
||||
}
|
||||
};
|
||||
let crypto_keys = CryptoKeys::init();
|
||||
let cr = CryptoRoot::create_pass(&password, &crypto_keys)?;
|
||||
println!("{}", cr.0);
|
||||
}
|
||||
CryptoRootCommand::NewClearText => {
|
||||
let crypto_keys = CryptoKeys::init();
|
||||
let cr = CryptoRoot::create_cleartext(&crypto_keys);
|
||||
println!("{}", cr.0);
|
||||
}
|
||||
CryptoRootCommand::ChangePassword {
|
||||
maybe_old_password,
|
||||
maybe_new_password,
|
||||
crypto_root,
|
||||
} => {
|
||||
let old_password = match maybe_old_password {
|
||||
Some(pwd) => pwd.to_string(),
|
||||
None => rpassword::prompt_password("Enter old password: ")?,
|
||||
};
|
||||
|
||||
let new_password = match maybe_new_password {
|
||||
Some(pwd) => pwd.to_string(),
|
||||
None => {
|
||||
let password = rpassword::prompt_password("Enter new password: ")?;
|
||||
let password_confirm =
|
||||
rpassword::prompt_password("Confirm new password: ")?;
|
||||
if password != password_confirm {
|
||||
bail!("Passwords don't match.");
|
||||
}
|
||||
password
|
||||
}
|
||||
};
|
||||
CryptoKeys::init(&creds, &user_secrets, &password).await?;
|
||||
|
||||
let keys = CryptoRoot(crypto_root.to_string()).crypto_keys(&old_password)?;
|
||||
let cr = CryptoRoot::create_pass(&new_password, &keys)?;
|
||||
println!("{}", cr.0);
|
||||
println!("");
|
||||
println!("Cryptographic key setup is complete.");
|
||||
println!("");
|
||||
println!("If you are using the static login provider, add the following section to your .toml configuration file:");
|
||||
println!("");
|
||||
dump_config(&password, &creds);
|
||||
}
|
||||
Command::InitializeLocalKeys { creds } => {
|
||||
let creds = make_storage_creds(creds);
|
||||
|
||||
println!("Please enter a password for local IMAP access.");
|
||||
println!("This password is not used for key decryption, your keys will be printed below (do not lose them!)");
|
||||
println!(
|
||||
"If you plan on using LDAP login, stop right here and use `first-login` instead"
|
||||
);
|
||||
let password = rpassword::prompt_password("Enter password: ")?;
|
||||
let password_confirm = rpassword::prompt_password("Confirm password: ")?;
|
||||
if password != password_confirm {
|
||||
bail!("Passwords don't match.");
|
||||
}
|
||||
|
||||
let master = gen_key();
|
||||
let (_, secret) = gen_keypair();
|
||||
let keys = CryptoKeys::init_without_password(&creds, &master, &secret).await?;
|
||||
|
||||
println!("");
|
||||
println!("Cryptographic key setup is complete.");
|
||||
println!("");
|
||||
println!("Add the following section to your .toml configuration file:");
|
||||
println!("");
|
||||
dump_config(&password, &creds);
|
||||
dump_keys(&keys);
|
||||
}
|
||||
Command::AddPassword {
|
||||
creds,
|
||||
user_secrets,
|
||||
gen,
|
||||
} => {
|
||||
let creds = make_storage_creds(creds);
|
||||
let user_secrets = make_user_secrets(user_secrets);
|
||||
|
||||
let existing_password =
|
||||
rpassword::prompt_password("Enter existing password to decrypt keys: ")?;
|
||||
let new_password = if gen {
|
||||
let password = base64::encode_config(
|
||||
&u128::to_be_bytes(thread_rng().gen())[..10],
|
||||
base64::URL_SAFE_NO_PAD,
|
||||
);
|
||||
println!("Your new password: {}", password);
|
||||
println!("Keep it safe!");
|
||||
password
|
||||
} else {
|
||||
let password = rpassword::prompt_password("Enter new password: ")?;
|
||||
let password_confirm = rpassword::prompt_password("Confirm new password: ")?;
|
||||
if password != password_confirm {
|
||||
bail!("Passwords don't match.");
|
||||
}
|
||||
CryptoRootCommand::DeriveIncoming { crypto_root } => {
|
||||
let pubkey = CryptoRoot(crypto_root.to_string()).public_key()?;
|
||||
let cr = CryptoRoot::create_incoming(&pubkey);
|
||||
println!("{}", cr.0);
|
||||
}
|
||||
},
|
||||
},
|
||||
password
|
||||
};
|
||||
|
||||
let keys = CryptoKeys::open(&creds, &user_secrets, &existing_password).await?;
|
||||
keys.add_password(&creds, &user_secrets, &new_password)
|
||||
.await?;
|
||||
println!("");
|
||||
println!("New password added successfully.");
|
||||
}
|
||||
Command::DeletePassword {
|
||||
creds,
|
||||
user_secrets,
|
||||
allow_delete_all,
|
||||
} => {
|
||||
let creds = make_storage_creds(creds);
|
||||
let user_secrets = make_user_secrets(user_secrets);
|
||||
|
||||
let existing_password = rpassword::prompt_password("Enter password to delete: ")?;
|
||||
|
||||
let keys = match allow_delete_all {
|
||||
true => Some(CryptoKeys::open(&creds, &user_secrets, &existing_password).await?),
|
||||
false => None,
|
||||
};
|
||||
|
||||
CryptoKeys::delete_password(&creds, &existing_password, allow_delete_all).await?;
|
||||
|
||||
println!("");
|
||||
println!("Password was deleted successfully.");
|
||||
|
||||
if let Some(keys) = keys {
|
||||
println!("As a reminder, here are your cryptographic keys:");
|
||||
dump_keys(&keys);
|
||||
}
|
||||
}
|
||||
Command::ShowKeys {
|
||||
creds,
|
||||
user_secrets,
|
||||
} => {
|
||||
let creds = make_storage_creds(creds);
|
||||
let user_secrets = make_user_secrets(user_secrets);
|
||||
|
||||
let existing_password = rpassword::prompt_password("Enter key decryption password: ")?;
|
||||
|
||||
let keys = CryptoKeys::open(&creds, &user_secrets, &existing_password).await?;
|
||||
dump_keys(&keys);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn reload(pid: Option<i32>, pid_path: Option<PathBuf>) -> Result<()> {
|
||||
let final_pid = match (pid, pid_path) {
|
||||
(Some(pid), _) => pid,
|
||||
(_, Some(path)) => {
|
||||
let mut f = std::fs::OpenOptions::new().read(true).open(path)?;
|
||||
let mut pidstr = String::new();
|
||||
f.read_to_string(&mut pidstr)?;
|
||||
pidstr.parse::<i32>()?
|
||||
}
|
||||
_ => bail!("Unable to infer your daemon's PID"),
|
||||
fn make_storage_creds(c: StorageCredsArgs) -> StorageCredentials {
|
||||
let s3_region = Region::Custom {
|
||||
name: c.region.clone(),
|
||||
endpoint: c.s3_endpoint,
|
||||
};
|
||||
let pid = Pid::from_raw(final_pid);
|
||||
signal::kill(pid, signal::Signal::SIGUSR1)?;
|
||||
Ok(())
|
||||
let k2v_region = Region::Custom {
|
||||
name: c.region,
|
||||
endpoint: c.k2v_endpoint,
|
||||
};
|
||||
StorageCredentials {
|
||||
k2v_region,
|
||||
s3_region,
|
||||
aws_access_key_id: c.aws_access_key_id,
|
||||
aws_secret_access_key: c.aws_secret_access_key,
|
||||
bucket: c.bucket,
|
||||
}
|
||||
}
|
||||
|
||||
fn account_management(root: &Command, cmd: &AccountManagement, users: PathBuf) -> Result<()> {
|
||||
let mut ulist: UserList =
|
||||
read_config(users.clone()).context(format!("'{:?}' must be a user database", users))?;
|
||||
|
||||
match cmd {
|
||||
AccountManagement::Add { login, setup } => {
|
||||
tracing::debug!(user = login, "will-create");
|
||||
let stp: SetupEntry = read_config(setup.clone())
|
||||
.context(format!("'{:?}' must be a setup file", setup))?;
|
||||
tracing::debug!(user = login, "loaded setup entry");
|
||||
|
||||
let password = match stp.clear_password {
|
||||
Some(pwd) => pwd,
|
||||
None => {
|
||||
let password = rpassword::prompt_password("Enter password: ")?;
|
||||
let password_confirm = rpassword::prompt_password("Confirm password: ")?;
|
||||
if password != password_confirm {
|
||||
bail!("Passwords don't match.");
|
||||
}
|
||||
password
|
||||
}
|
||||
};
|
||||
|
||||
let crypto_keys = CryptoKeys::init();
|
||||
let crypto_root = match root {
|
||||
Command::Provider(_) => CryptoRoot::create_pass(&password, &crypto_keys)?,
|
||||
Command::Companion(_) => CryptoRoot::create_cleartext(&crypto_keys),
|
||||
_ => unreachable!(),
|
||||
};
|
||||
|
||||
let hash = hash_password(password.as_str()).context("unable to hash password")?;
|
||||
|
||||
ulist.insert(
|
||||
login.clone(),
|
||||
UserEntry {
|
||||
email_addresses: stp.email_addresses,
|
||||
password: hash,
|
||||
crypto_root: crypto_root.0,
|
||||
storage: stp.storage,
|
||||
},
|
||||
);
|
||||
|
||||
write_config(users.clone(), &ulist)?;
|
||||
}
|
||||
AccountManagement::Delete { login } => {
|
||||
tracing::debug!(user = login, "will-delete");
|
||||
ulist.remove(login);
|
||||
write_config(users.clone(), &ulist)?;
|
||||
}
|
||||
AccountManagement::ChangePassword {
|
||||
maybe_old_password,
|
||||
maybe_new_password,
|
||||
login,
|
||||
} => {
|
||||
let mut user = ulist.remove(login).context("user must exist first")?;
|
||||
|
||||
let old_password = match maybe_old_password {
|
||||
Some(pwd) => pwd.to_string(),
|
||||
None => rpassword::prompt_password("Enter old password: ")?,
|
||||
};
|
||||
|
||||
if !verify_password(&old_password, &user.password)? {
|
||||
bail!(format!("invalid password for login {}", login));
|
||||
}
|
||||
|
||||
let crypto_keys = CryptoRoot(user.crypto_root).crypto_keys(&old_password)?;
|
||||
|
||||
let new_password = match maybe_new_password {
|
||||
Some(pwd) => pwd.to_string(),
|
||||
None => {
|
||||
let password = rpassword::prompt_password("Enter new password: ")?;
|
||||
let password_confirm = rpassword::prompt_password("Confirm new password: ")?;
|
||||
if password != password_confirm {
|
||||
bail!("Passwords don't match.");
|
||||
}
|
||||
password
|
||||
}
|
||||
};
|
||||
let new_hash = hash_password(&new_password)?;
|
||||
let new_crypto_root = CryptoRoot::create_pass(&new_password, &crypto_keys)?;
|
||||
|
||||
user.password = new_hash;
|
||||
user.crypto_root = new_crypto_root.0;
|
||||
|
||||
ulist.insert(login.clone(), user);
|
||||
write_config(users.clone(), &ulist)?;
|
||||
}
|
||||
};
|
||||
|
||||
Ok(())
|
||||
fn make_user_secrets(c: UserSecretsArgs) -> UserSecrets {
|
||||
UserSecrets {
|
||||
user_secret: c.user_secret,
|
||||
alternate_user_secrets: c
|
||||
.alternate_user_secrets
|
||||
.split(",")
|
||||
.map(|x| x.trim())
|
||||
.filter(|x| !x.is_empty())
|
||||
.map(|x| x.to_string())
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn dump_config(password: &str, creds: &StorageCredentials) {
|
||||
println!("[login_static.users.<username>]");
|
||||
println!(
|
||||
"password = \"{}\"",
|
||||
hash_password(password).expect("unable to hash password")
|
||||
);
|
||||
println!("aws_access_key_id = \"{}\"", creds.aws_access_key_id);
|
||||
println!(
|
||||
"aws_secret_access_key = \"{}\"",
|
||||
creds.aws_secret_access_key
|
||||
);
|
||||
}
|
||||
|
||||
fn dump_keys(keys: &CryptoKeys) {
|
||||
println!("master_key = \"{}\"", base64::encode(&keys.master));
|
||||
println!("secret_key = \"{}\"", base64::encode(&keys.secret));
|
||||
}
|
||||
|
|
|
|||
104
src/server.rs
104
src/server.rs
|
|
@ -1,89 +1,43 @@
|
|||
use std::io::Write;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::Result;
|
||||
use futures::try_join;
|
||||
use anyhow::{bail, Result};
|
||||
use futures::{try_join, StreamExt};
|
||||
use log::*;
|
||||
use rusoto_signature::Region;
|
||||
use tokio::sync::watch;
|
||||
|
||||
use crate::auth;
|
||||
use crate::config::*;
|
||||
use crate::imap;
|
||||
use crate::lmtp::*;
|
||||
use crate::login::ArcLoginProvider;
|
||||
use crate::login::{demo_provider::*, ldap_provider::*, static_provider::*};
|
||||
use crate::login::{ldap_provider::*, static_provider::*, *};
|
||||
|
||||
pub struct Server {
|
||||
lmtp_server: Option<Arc<LmtpServer>>,
|
||||
imap_unsecure_server: Option<imap::Server>,
|
||||
imap_server: Option<imap::Server>,
|
||||
auth_server: Option<auth::AuthServer>,
|
||||
pid_file: Option<PathBuf>,
|
||||
}
|
||||
|
||||
impl Server {
|
||||
pub async fn from_companion_config(config: CompanionConfig) -> Result<Self> {
|
||||
tracing::info!("Init as companion");
|
||||
let login = Arc::new(StaticLoginProvider::new(config.users).await?);
|
||||
pub async fn new(config: Config) -> Result<Self> {
|
||||
let (login, lmtp_conf, imap_conf) = build(config)?;
|
||||
|
||||
let lmtp_server = None;
|
||||
let imap_unsecure_server = Some(imap::new_unsecure(config.imap, login.clone()));
|
||||
Ok(Self {
|
||||
lmtp_server,
|
||||
imap_unsecure_server,
|
||||
imap_server: None,
|
||||
auth_server: None,
|
||||
pid_file: config.pid,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn from_provider_config(config: ProviderConfig) -> Result<Self> {
|
||||
tracing::info!("Init as provider");
|
||||
let login: ArcLoginProvider = match config.users {
|
||||
UserManagement::Demo => Arc::new(DemoLoginProvider::new()),
|
||||
UserManagement::Static(x) => Arc::new(StaticLoginProvider::new(x).await?),
|
||||
UserManagement::Ldap(x) => Arc::new(LdapLoginProvider::new(x)?),
|
||||
let lmtp_server = lmtp_conf.map(|cfg| LmtpServer::new(cfg, login.clone()));
|
||||
let imap_server = match imap_conf {
|
||||
Some(cfg) => Some(imap::new(cfg, login.clone()).await?),
|
||||
None => None,
|
||||
};
|
||||
|
||||
let lmtp_server = config.lmtp.map(|lmtp| LmtpServer::new(lmtp, login.clone()));
|
||||
let imap_unsecure_server = config
|
||||
.imap_unsecure
|
||||
.map(|imap| imap::new_unsecure(imap, login.clone()));
|
||||
let imap_server = config
|
||||
.imap
|
||||
.map(|imap| imap::new(imap, login.clone()))
|
||||
.transpose()?;
|
||||
let auth_server = config
|
||||
.auth
|
||||
.map(|auth| auth::AuthServer::new(auth, login.clone()));
|
||||
|
||||
Ok(Self {
|
||||
lmtp_server,
|
||||
imap_unsecure_server,
|
||||
imap_server,
|
||||
auth_server,
|
||||
pid_file: config.pid,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn run(self) -> Result<()> {
|
||||
let pid = std::process::id();
|
||||
tracing::info!(pid = pid, "Starting main loops");
|
||||
|
||||
// write the pid file
|
||||
if let Some(pid_file) = self.pid_file {
|
||||
let mut file = std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.open(pid_file)?;
|
||||
file.write_all(pid.to_string().as_bytes())?;
|
||||
drop(file);
|
||||
}
|
||||
tracing::info!("Starting Aerogramme...");
|
||||
|
||||
let (exit_signal, provoke_exit) = watch_ctrl_c();
|
||||
let _exit_on_err = move |err: anyhow::Error| {
|
||||
let exit_on_err = move |err: anyhow::Error| {
|
||||
error!("Error: {}", err);
|
||||
let _ = provoke_exit.send(true);
|
||||
};
|
||||
|
|
@ -95,23 +49,11 @@ impl Server {
|
|||
Some(s) => s.run(exit_signal.clone()).await,
|
||||
}
|
||||
},
|
||||
async {
|
||||
match self.imap_unsecure_server {
|
||||
None => Ok(()),
|
||||
Some(s) => s.run(exit_signal.clone()).await,
|
||||
}
|
||||
},
|
||||
async {
|
||||
match self.imap_server {
|
||||
None => Ok(()),
|
||||
Some(s) => s.run(exit_signal.clone()).await,
|
||||
}
|
||||
},
|
||||
async {
|
||||
match self.auth_server {
|
||||
None => Ok(()),
|
||||
Some(a) => a.run(exit_signal.clone()).await,
|
||||
}
|
||||
}
|
||||
)?;
|
||||
|
||||
|
|
@ -119,6 +61,28 @@ impl Server {
|
|||
}
|
||||
}
|
||||
|
||||
fn build(config: Config) -> Result<(ArcLoginProvider, Option<LmtpConfig>, Option<ImapConfig>)> {
|
||||
let s3_region = Region::Custom {
|
||||
name: config.aws_region.clone(),
|
||||
endpoint: config.s3_endpoint,
|
||||
};
|
||||
let k2v_region = Region::Custom {
|
||||
name: config.aws_region,
|
||||
endpoint: config.k2v_endpoint,
|
||||
};
|
||||
|
||||
let lp: ArcLoginProvider = match (config.login_static, config.login_ldap) {
|
||||
(Some(st), None) => Arc::new(StaticLoginProvider::new(st, k2v_region, s3_region)?),
|
||||
(None, Some(ld)) => Arc::new(LdapLoginProvider::new(ld, k2v_region, s3_region)?),
|
||||
(Some(_), Some(_)) => {
|
||||
bail!("A single login provider must be set up in config file")
|
||||
}
|
||||
(None, None) => bail!("No login provider is set up in config file"),
|
||||
};
|
||||
|
||||
Ok((lp, config.lmtp, config.imap))
|
||||
}
|
||||
|
||||
pub fn watch_ctrl_c() -> (watch::Receiver<bool>, Arc<watch::Sender<bool>>) {
|
||||
let (send_cancel, watch_cancel) = watch::channel(false);
|
||||
let send_cancel = Arc::new(send_cancel);
|
||||
|
|
|
|||
|
|
@ -1,538 +0,0 @@
|
|||
use aws_sdk_s3::{self as s3, error::SdkError, operation::get_object::GetObjectError};
|
||||
use aws_smithy_runtime::client::http::hyper_014::HyperClientBuilder;
|
||||
use aws_smithy_runtime_api::client::http::SharedHttpClient;
|
||||
use hyper_rustls::HttpsConnector;
|
||||
use hyper_util::client::legacy::{connect::HttpConnector, Client as HttpClient};
|
||||
use hyper_util::rt::TokioExecutor;
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::storage::*;
|
||||
|
||||
pub struct GarageRoot {
|
||||
k2v_http: HttpClient<HttpsConnector<HttpConnector>, k2v_client::Body>,
|
||||
aws_http: SharedHttpClient,
|
||||
}
|
||||
|
||||
impl GarageRoot {
|
||||
pub fn new() -> anyhow::Result<Self> {
|
||||
let connector = hyper_rustls::HttpsConnectorBuilder::new()
|
||||
.with_native_roots()?
|
||||
.https_or_http()
|
||||
.enable_http1()
|
||||
.enable_http2()
|
||||
.build();
|
||||
let k2v_http = HttpClient::builder(TokioExecutor::new()).build(connector);
|
||||
let aws_http = HyperClientBuilder::new().build_https();
|
||||
Ok(Self { k2v_http, aws_http })
|
||||
}
|
||||
|
||||
pub fn user(&self, conf: GarageConf) -> anyhow::Result<Arc<GarageUser>> {
|
||||
let mut unicity: Vec<u8> = vec![];
|
||||
unicity.extend_from_slice(file!().as_bytes());
|
||||
unicity.append(&mut rmp_serde::to_vec(&conf)?);
|
||||
|
||||
Ok(Arc::new(GarageUser {
|
||||
conf,
|
||||
aws_http: self.aws_http.clone(),
|
||||
k2v_http: self.k2v_http.clone(),
|
||||
unicity,
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct GarageConf {
|
||||
pub region: String,
|
||||
pub s3_endpoint: String,
|
||||
pub k2v_endpoint: String,
|
||||
pub aws_access_key_id: String,
|
||||
pub aws_secret_access_key: String,
|
||||
pub bucket: String,
|
||||
}
|
||||
|
||||
//@FIXME we should get rid of this builder
|
||||
//and allocate a S3 + K2V client only once per user
|
||||
//(and using a shared HTTP client)
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct GarageUser {
|
||||
conf: GarageConf,
|
||||
aws_http: SharedHttpClient,
|
||||
k2v_http: HttpClient<HttpsConnector<HttpConnector>, k2v_client::Body>,
|
||||
unicity: Vec<u8>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl IBuilder for GarageUser {
|
||||
async fn build(&self) -> Result<Store, StorageError> {
|
||||
let s3_creds = s3::config::Credentials::new(
|
||||
self.conf.aws_access_key_id.clone(),
|
||||
self.conf.aws_secret_access_key.clone(),
|
||||
None,
|
||||
None,
|
||||
"aerogramme",
|
||||
);
|
||||
|
||||
let sdk_config = aws_config::from_env()
|
||||
.region(aws_config::Region::new(self.conf.region.clone()))
|
||||
.credentials_provider(s3_creds)
|
||||
.http_client(self.aws_http.clone())
|
||||
.endpoint_url(self.conf.s3_endpoint.clone())
|
||||
.load()
|
||||
.await;
|
||||
|
||||
let s3_config = aws_sdk_s3::config::Builder::from(&sdk_config)
|
||||
.force_path_style(true)
|
||||
.build();
|
||||
|
||||
let s3_client = aws_sdk_s3::Client::from_conf(s3_config);
|
||||
|
||||
let k2v_config = k2v_client::K2vClientConfig {
|
||||
endpoint: self.conf.k2v_endpoint.clone(),
|
||||
region: self.conf.region.clone(),
|
||||
aws_access_key_id: self.conf.aws_access_key_id.clone(),
|
||||
aws_secret_access_key: self.conf.aws_secret_access_key.clone(),
|
||||
bucket: self.conf.bucket.clone(),
|
||||
user_agent: None,
|
||||
};
|
||||
|
||||
let k2v_client =
|
||||
match k2v_client::K2vClient::new_with_client(k2v_config, self.k2v_http.clone()) {
|
||||
Err(e) => {
|
||||
tracing::error!("unable to build k2v client: {}", e);
|
||||
return Err(StorageError::Internal);
|
||||
}
|
||||
Ok(v) => v,
|
||||
};
|
||||
|
||||
Ok(Box::new(GarageStore {
|
||||
bucket: self.conf.bucket.clone(),
|
||||
s3: s3_client,
|
||||
k2v: k2v_client,
|
||||
}))
|
||||
}
|
||||
fn unique(&self) -> UnicityBuffer {
|
||||
UnicityBuffer(self.unicity.clone())
|
||||
}
|
||||
}
|
||||
|
||||
pub struct GarageStore {
|
||||
bucket: String,
|
||||
s3: s3::Client,
|
||||
k2v: k2v_client::K2vClient,
|
||||
}
|
||||
|
||||
fn causal_to_row_val(row_ref: RowRef, causal_value: k2v_client::CausalValue) -> RowVal {
|
||||
let new_row_ref = row_ref.with_causality(causal_value.causality.into());
|
||||
let row_values = causal_value
|
||||
.value
|
||||
.into_iter()
|
||||
.map(|k2v_value| match k2v_value {
|
||||
k2v_client::K2vValue::Tombstone => Alternative::Tombstone,
|
||||
k2v_client::K2vValue::Value(v) => Alternative::Value(v),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
RowVal {
|
||||
row_ref: new_row_ref,
|
||||
value: row_values,
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl IStore for GarageStore {
|
||||
async fn row_fetch<'a>(&self, select: &Selector<'a>) -> Result<Vec<RowVal>, StorageError> {
|
||||
tracing::trace!(select=%select, command="row_fetch");
|
||||
let (pk_list, batch_op) = match select {
|
||||
Selector::Range {
|
||||
shard,
|
||||
sort_begin,
|
||||
sort_end,
|
||||
} => (
|
||||
vec![shard.to_string()],
|
||||
vec![k2v_client::BatchReadOp {
|
||||
partition_key: shard,
|
||||
filter: k2v_client::Filter {
|
||||
start: Some(sort_begin),
|
||||
end: Some(sort_end),
|
||||
..k2v_client::Filter::default()
|
||||
},
|
||||
..k2v_client::BatchReadOp::default()
|
||||
}],
|
||||
),
|
||||
Selector::List(row_ref_list) => (
|
||||
row_ref_list
|
||||
.iter()
|
||||
.map(|row_ref| row_ref.uid.shard.to_string())
|
||||
.collect::<Vec<_>>(),
|
||||
row_ref_list
|
||||
.iter()
|
||||
.map(|row_ref| k2v_client::BatchReadOp {
|
||||
partition_key: &row_ref.uid.shard,
|
||||
filter: k2v_client::Filter {
|
||||
start: Some(&row_ref.uid.sort),
|
||||
..k2v_client::Filter::default()
|
||||
},
|
||||
single_item: true,
|
||||
..k2v_client::BatchReadOp::default()
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
),
|
||||
Selector::Prefix { shard, sort_prefix } => (
|
||||
vec![shard.to_string()],
|
||||
vec![k2v_client::BatchReadOp {
|
||||
partition_key: shard,
|
||||
filter: k2v_client::Filter {
|
||||
prefix: Some(sort_prefix),
|
||||
..k2v_client::Filter::default()
|
||||
},
|
||||
..k2v_client::BatchReadOp::default()
|
||||
}],
|
||||
),
|
||||
Selector::Single(row_ref) => {
|
||||
let causal_value = match self
|
||||
.k2v
|
||||
.read_item(&row_ref.uid.shard, &row_ref.uid.sort)
|
||||
.await
|
||||
{
|
||||
Err(k2v_client::Error::NotFound) => {
|
||||
tracing::debug!(
|
||||
"K2V item not found shard={}, sort={}, bucket={}",
|
||||
row_ref.uid.shard,
|
||||
row_ref.uid.sort,
|
||||
self.bucket,
|
||||
);
|
||||
return Err(StorageError::NotFound);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!(
|
||||
"K2V read item shard={}, sort={}, bucket={} failed: {}",
|
||||
row_ref.uid.shard,
|
||||
row_ref.uid.sort,
|
||||
self.bucket,
|
||||
e
|
||||
);
|
||||
return Err(StorageError::Internal);
|
||||
}
|
||||
Ok(v) => v,
|
||||
};
|
||||
|
||||
let row_val = causal_to_row_val((*row_ref).clone(), causal_value);
|
||||
return Ok(vec![row_val]);
|
||||
}
|
||||
};
|
||||
|
||||
let all_raw_res = match self.k2v.read_batch(&batch_op).await {
|
||||
Err(e) => {
|
||||
tracing::error!(
|
||||
"k2v read batch failed for {:?}, bucket {} with err: {}",
|
||||
select,
|
||||
self.bucket,
|
||||
e
|
||||
);
|
||||
return Err(StorageError::Internal);
|
||||
}
|
||||
Ok(v) => v,
|
||||
};
|
||||
//println!("fetch res -> {:?}", all_raw_res);
|
||||
|
||||
let row_vals =
|
||||
all_raw_res
|
||||
.into_iter()
|
||||
.zip(pk_list.into_iter())
|
||||
.fold(vec![], |mut acc, (page, pk)| {
|
||||
page.items
|
||||
.into_iter()
|
||||
.map(|(sk, cv)| causal_to_row_val(RowRef::new(&pk, &sk), cv))
|
||||
.for_each(|rr| acc.push(rr));
|
||||
|
||||
acc
|
||||
});
|
||||
tracing::debug!(fetch_count = row_vals.len(), command = "row_fetch");
|
||||
|
||||
Ok(row_vals)
|
||||
}
|
||||
async fn row_rm<'a>(&self, select: &Selector<'a>) -> Result<(), StorageError> {
|
||||
tracing::trace!(select=%select, command="row_rm");
|
||||
let del_op = match select {
|
||||
Selector::Range {
|
||||
shard,
|
||||
sort_begin,
|
||||
sort_end,
|
||||
} => vec![k2v_client::BatchDeleteOp {
|
||||
partition_key: shard,
|
||||
prefix: None,
|
||||
start: Some(sort_begin),
|
||||
end: Some(sort_end),
|
||||
single_item: false,
|
||||
}],
|
||||
Selector::List(row_ref_list) => {
|
||||
// Insert null values with causality token = delete
|
||||
let batch_op = row_ref_list
|
||||
.iter()
|
||||
.map(|v| k2v_client::BatchInsertOp {
|
||||
partition_key: &v.uid.shard,
|
||||
sort_key: &v.uid.sort,
|
||||
causality: v.causality.clone().map(|ct| ct.into()),
|
||||
value: k2v_client::K2vValue::Tombstone,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
return match self.k2v.insert_batch(&batch_op).await {
|
||||
Err(e) => {
|
||||
tracing::error!("Unable to delete the list of values: {}", e);
|
||||
Err(StorageError::Internal)
|
||||
}
|
||||
Ok(_) => Ok(()),
|
||||
};
|
||||
}
|
||||
Selector::Prefix { shard, sort_prefix } => vec![k2v_client::BatchDeleteOp {
|
||||
partition_key: shard,
|
||||
prefix: Some(sort_prefix),
|
||||
start: None,
|
||||
end: None,
|
||||
single_item: false,
|
||||
}],
|
||||
Selector::Single(row_ref) => {
|
||||
// Insert null values with causality token = delete
|
||||
let batch_op = vec![k2v_client::BatchInsertOp {
|
||||
partition_key: &row_ref.uid.shard,
|
||||
sort_key: &row_ref.uid.sort,
|
||||
causality: row_ref.causality.clone().map(|ct| ct.into()),
|
||||
value: k2v_client::K2vValue::Tombstone,
|
||||
}];
|
||||
|
||||
return match self.k2v.insert_batch(&batch_op).await {
|
||||
Err(e) => {
|
||||
tracing::error!("Unable to delete the list of values: {}", e);
|
||||
Err(StorageError::Internal)
|
||||
}
|
||||
Ok(_) => Ok(()),
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
// Finally here we only have prefix & range
|
||||
match self.k2v.delete_batch(&del_op).await {
|
||||
Err(e) => {
|
||||
tracing::error!("delete batch error: {}", e);
|
||||
Err(StorageError::Internal)
|
||||
}
|
||||
Ok(_) => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
async fn row_insert(&self, values: Vec<RowVal>) -> Result<(), StorageError> {
|
||||
tracing::trace!(entries=%values.iter().map(|v| v.row_ref.to_string()).collect::<Vec<_>>().join(","), command="row_insert");
|
||||
let batch_ops = values
|
||||
.iter()
|
||||
.map(|v| k2v_client::BatchInsertOp {
|
||||
partition_key: &v.row_ref.uid.shard,
|
||||
sort_key: &v.row_ref.uid.sort,
|
||||
causality: v.row_ref.causality.clone().map(|ct| ct.into()),
|
||||
value: v
|
||||
.value
|
||||
.iter()
|
||||
.next()
|
||||
.map(|cv| match cv {
|
||||
Alternative::Value(buff) => k2v_client::K2vValue::Value(buff.clone()),
|
||||
Alternative::Tombstone => k2v_client::K2vValue::Tombstone,
|
||||
})
|
||||
.unwrap_or(k2v_client::K2vValue::Tombstone),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
match self.k2v.insert_batch(&batch_ops).await {
|
||||
Err(e) => {
|
||||
tracing::error!("k2v can't insert some value: {}", e);
|
||||
Err(StorageError::Internal)
|
||||
}
|
||||
Ok(v) => Ok(v),
|
||||
}
|
||||
}
|
||||
async fn row_poll(&self, value: &RowRef) -> Result<RowVal, StorageError> {
|
||||
tracing::trace!(entry=%value, command="row_poll");
|
||||
loop {
|
||||
if let Some(ct) = &value.causality {
|
||||
match self
|
||||
.k2v
|
||||
.poll_item(&value.uid.shard, &value.uid.sort, ct.clone().into(), None)
|
||||
.await
|
||||
{
|
||||
Err(e) => {
|
||||
tracing::error!("Unable to poll item: {}", e);
|
||||
return Err(StorageError::Internal);
|
||||
}
|
||||
Ok(None) => continue,
|
||||
Ok(Some(cv)) => return Ok(causal_to_row_val(value.clone(), cv)),
|
||||
}
|
||||
} else {
|
||||
match self.k2v.read_item(&value.uid.shard, &value.uid.sort).await {
|
||||
Err(k2v_client::Error::NotFound) => {
|
||||
self.k2v
|
||||
.insert_item(&value.uid.shard, &value.uid.sort, vec![0u8], None)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
tracing::error!("Unable to insert item in polling logic: {}", e);
|
||||
StorageError::Internal
|
||||
})?;
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!("Unable to read item in polling logic: {}", e);
|
||||
return Err(StorageError::Internal);
|
||||
}
|
||||
Ok(cv) => return Ok(causal_to_row_val(value.clone(), cv)),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn blob_fetch(&self, blob_ref: &BlobRef) -> Result<BlobVal, StorageError> {
|
||||
tracing::trace!(entry=%blob_ref, command="blob_fetch");
|
||||
let maybe_out = self
|
||||
.s3
|
||||
.get_object()
|
||||
.bucket(self.bucket.to_string())
|
||||
.key(blob_ref.0.to_string())
|
||||
.send()
|
||||
.await;
|
||||
|
||||
let object_output = match maybe_out {
|
||||
Ok(output) => output,
|
||||
Err(SdkError::ServiceError(x)) => match x.err() {
|
||||
GetObjectError::NoSuchKey(_) => return Err(StorageError::NotFound),
|
||||
e => {
|
||||
tracing::warn!("Blob Fetch Error, Service Error: {}", e);
|
||||
return Err(StorageError::Internal);
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
tracing::warn!("Blob Fetch Error, {}", e);
|
||||
return Err(StorageError::Internal);
|
||||
}
|
||||
};
|
||||
|
||||
let buffer = match object_output.body.collect().await {
|
||||
Ok(aggreg) => aggreg.to_vec(),
|
||||
Err(e) => {
|
||||
tracing::warn!("Fetching body failed with {}", e);
|
||||
return Err(StorageError::Internal);
|
||||
}
|
||||
};
|
||||
|
||||
let mut bv = BlobVal::new(blob_ref.clone(), buffer);
|
||||
if let Some(meta) = object_output.metadata {
|
||||
bv.meta = meta;
|
||||
}
|
||||
tracing::debug!("Fetched {}/{}", self.bucket, blob_ref.0);
|
||||
Ok(bv)
|
||||
}
|
||||
async fn blob_insert(&self, blob_val: BlobVal) -> Result<(), StorageError> {
|
||||
tracing::trace!(entry=%blob_val.blob_ref, command="blob_insert");
|
||||
let streamable_value = s3::primitives::ByteStream::from(blob_val.value);
|
||||
|
||||
let maybe_send = self
|
||||
.s3
|
||||
.put_object()
|
||||
.bucket(self.bucket.to_string())
|
||||
.key(blob_val.blob_ref.0.to_string())
|
||||
.set_metadata(Some(blob_val.meta))
|
||||
.body(streamable_value)
|
||||
.send()
|
||||
.await;
|
||||
|
||||
match maybe_send {
|
||||
Err(e) => {
|
||||
tracing::error!("unable to send object: {}", e);
|
||||
Err(StorageError::Internal)
|
||||
}
|
||||
Ok(_) => {
|
||||
tracing::debug!("Inserted {}/{}", self.bucket, blob_val.blob_ref.0);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
async fn blob_copy(&self, src: &BlobRef, dst: &BlobRef) -> Result<(), StorageError> {
|
||||
tracing::trace!(src=%src, dst=%dst, command="blob_copy");
|
||||
let maybe_copy = self
|
||||
.s3
|
||||
.copy_object()
|
||||
.bucket(self.bucket.to_string())
|
||||
.key(dst.0.clone())
|
||||
.copy_source(format!("/{}/{}", self.bucket.to_string(), src.0.clone()))
|
||||
.send()
|
||||
.await;
|
||||
|
||||
match maybe_copy {
|
||||
Err(e) => {
|
||||
tracing::error!(
|
||||
"unable to copy object {} to {} (bucket: {}), error: {}",
|
||||
src.0,
|
||||
dst.0,
|
||||
self.bucket,
|
||||
e
|
||||
);
|
||||
Err(StorageError::Internal)
|
||||
}
|
||||
Ok(_) => {
|
||||
tracing::debug!("copied {} to {} (bucket: {})", src.0, dst.0, self.bucket);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
async fn blob_list(&self, prefix: &str) -> Result<Vec<BlobRef>, StorageError> {
|
||||
tracing::trace!(prefix = prefix, command = "blob_list");
|
||||
let maybe_list = self
|
||||
.s3
|
||||
.list_objects_v2()
|
||||
.bucket(self.bucket.to_string())
|
||||
.prefix(prefix)
|
||||
.into_paginator()
|
||||
.send()
|
||||
.try_collect()
|
||||
.await;
|
||||
|
||||
match maybe_list {
|
||||
Err(e) => {
|
||||
tracing::error!(
|
||||
"listing prefix {} on bucket {} failed: {}",
|
||||
prefix,
|
||||
self.bucket,
|
||||
e
|
||||
);
|
||||
Err(StorageError::Internal)
|
||||
}
|
||||
Ok(pagin_list_out) => Ok(pagin_list_out
|
||||
.into_iter()
|
||||
.map(|list_out| list_out.contents.unwrap_or(vec![]))
|
||||
.flatten()
|
||||
.map(|obj| BlobRef(obj.key.unwrap_or(String::new())))
|
||||
.collect::<Vec<_>>()),
|
||||
}
|
||||
}
|
||||
async fn blob_rm(&self, blob_ref: &BlobRef) -> Result<(), StorageError> {
|
||||
tracing::trace!(entry=%blob_ref, command="blob_rm");
|
||||
let maybe_delete = self
|
||||
.s3
|
||||
.delete_object()
|
||||
.bucket(self.bucket.to_string())
|
||||
.key(blob_ref.0.clone())
|
||||
.send()
|
||||
.await;
|
||||
|
||||
match maybe_delete {
|
||||
Err(e) => {
|
||||
tracing::error!(
|
||||
"unable to delete {} (bucket: {}), error {}",
|
||||
blob_ref.0,
|
||||
self.bucket,
|
||||
e
|
||||
);
|
||||
Err(StorageError::Internal)
|
||||
}
|
||||
Ok(_) => {
|
||||
tracing::debug!("deleted {} (bucket: {})", blob_ref.0, self.bucket);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,334 +0,0 @@
|
|||
use crate::storage::*;
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::ops::Bound::{self, Excluded, Included, Unbounded};
|
||||
use std::sync::{Arc, RwLock};
|
||||
use tokio::sync::Notify;
|
||||
|
||||
/// This implementation is very inneficient, and not completely correct
|
||||
/// Indeed, when the connector is dropped, the memory is freed.
|
||||
/// It means that when a user disconnects, its data are lost.
|
||||
/// It's intended only for basic debugging, do not use it for advanced tests...
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct MemDb(tokio::sync::Mutex<HashMap<String, Arc<MemBuilder>>>);
|
||||
impl MemDb {
|
||||
pub fn new() -> Self {
|
||||
Self(tokio::sync::Mutex::new(HashMap::new()))
|
||||
}
|
||||
|
||||
pub async fn builder(&self, username: &str) -> Arc<MemBuilder> {
|
||||
let mut global_storage = self.0.lock().await;
|
||||
global_storage
|
||||
.entry(username.to_string())
|
||||
.or_insert(MemBuilder::new(username))
|
||||
.clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
enum InternalData {
|
||||
Tombstone,
|
||||
Value(Vec<u8>),
|
||||
}
|
||||
impl InternalData {
|
||||
fn to_alternative(&self) -> Alternative {
|
||||
match self {
|
||||
Self::Tombstone => Alternative::Tombstone,
|
||||
Self::Value(x) => Alternative::Value(x.clone()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct InternalRowVal {
|
||||
data: Vec<InternalData>,
|
||||
version: u64,
|
||||
change: Arc<Notify>,
|
||||
}
|
||||
impl std::default::Default for InternalRowVal {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
data: vec![],
|
||||
version: 1,
|
||||
change: Arc::new(Notify::new()),
|
||||
}
|
||||
}
|
||||
}
|
||||
impl InternalRowVal {
|
||||
fn concurrent_values(&self) -> Vec<Alternative> {
|
||||
self.data.iter().map(InternalData::to_alternative).collect()
|
||||
}
|
||||
|
||||
fn to_row_val(&self, row_ref: RowRef) -> RowVal {
|
||||
RowVal {
|
||||
row_ref: row_ref.with_causality(self.version.to_string()),
|
||||
value: self.concurrent_values(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone)]
|
||||
struct InternalBlobVal {
|
||||
data: Vec<u8>,
|
||||
metadata: HashMap<String, String>,
|
||||
}
|
||||
impl InternalBlobVal {
|
||||
fn to_blob_val(&self, bref: &BlobRef) -> BlobVal {
|
||||
BlobVal {
|
||||
blob_ref: bref.clone(),
|
||||
meta: self.metadata.clone(),
|
||||
value: self.data.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type ArcRow = Arc<RwLock<HashMap<String, BTreeMap<String, InternalRowVal>>>>;
|
||||
type ArcBlob = Arc<RwLock<BTreeMap<String, InternalBlobVal>>>;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct MemBuilder {
|
||||
unicity: Vec<u8>,
|
||||
row: ArcRow,
|
||||
blob: ArcBlob,
|
||||
}
|
||||
|
||||
impl MemBuilder {
|
||||
pub fn new(user: &str) -> Arc<Self> {
|
||||
tracing::debug!("initialize membuilder for {}", user);
|
||||
let mut unicity: Vec<u8> = vec![];
|
||||
unicity.extend_from_slice(file!().as_bytes());
|
||||
unicity.extend_from_slice(user.as_bytes());
|
||||
Arc::new(Self {
|
||||
unicity,
|
||||
row: Arc::new(RwLock::new(HashMap::new())),
|
||||
blob: Arc::new(RwLock::new(BTreeMap::new())),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl IBuilder for MemBuilder {
|
||||
async fn build(&self) -> Result<Store, StorageError> {
|
||||
Ok(Box::new(MemStore {
|
||||
row: self.row.clone(),
|
||||
blob: self.blob.clone(),
|
||||
}))
|
||||
}
|
||||
|
||||
fn unique(&self) -> UnicityBuffer {
|
||||
UnicityBuffer(self.unicity.clone())
|
||||
}
|
||||
}
|
||||
|
||||
pub struct MemStore {
|
||||
row: ArcRow,
|
||||
blob: ArcBlob,
|
||||
}
|
||||
|
||||
fn prefix_last_bound(prefix: &str) -> Bound<String> {
|
||||
let mut sort_end = prefix.to_string();
|
||||
match sort_end.pop() {
|
||||
None => Unbounded,
|
||||
Some(ch) => {
|
||||
let nc = char::from_u32(ch as u32 + 1).unwrap();
|
||||
sort_end.push(nc);
|
||||
Excluded(sort_end)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl MemStore {
|
||||
fn row_rm_single(&self, entry: &RowRef) -> Result<(), StorageError> {
|
||||
tracing::trace!(entry=%entry, command="row_rm_single");
|
||||
let mut store = self.row.write().or(Err(StorageError::Internal))?;
|
||||
let shard = &entry.uid.shard;
|
||||
let sort = &entry.uid.sort;
|
||||
|
||||
let cauz = match entry.causality.as_ref().map(|v| v.parse::<u64>()) {
|
||||
Some(Ok(v)) => v,
|
||||
_ => 0,
|
||||
};
|
||||
|
||||
let bt = store.entry(shard.to_string()).or_default();
|
||||
let intval = bt.entry(sort.to_string()).or_default();
|
||||
|
||||
if cauz == intval.version {
|
||||
intval.data.clear();
|
||||
}
|
||||
intval.data.push(InternalData::Tombstone);
|
||||
intval.version += 1;
|
||||
intval.change.notify_waiters();
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl IStore for MemStore {
|
||||
async fn row_fetch<'a>(&self, select: &Selector<'a>) -> Result<Vec<RowVal>, StorageError> {
|
||||
tracing::trace!(select=%select, command="row_fetch");
|
||||
let store = self.row.read().or(Err(StorageError::Internal))?;
|
||||
|
||||
match select {
|
||||
Selector::Range {
|
||||
shard,
|
||||
sort_begin,
|
||||
sort_end,
|
||||
} => Ok(store
|
||||
.get(*shard)
|
||||
.unwrap_or(&BTreeMap::new())
|
||||
.range((
|
||||
Included(sort_begin.to_string()),
|
||||
Excluded(sort_end.to_string()),
|
||||
))
|
||||
.map(|(k, v)| v.to_row_val(RowRef::new(shard, k)))
|
||||
.collect::<Vec<_>>()),
|
||||
Selector::List(rlist) => {
|
||||
let mut acc = vec![];
|
||||
for row_ref in rlist {
|
||||
let maybe_intval = store
|
||||
.get(&row_ref.uid.shard)
|
||||
.map(|v| v.get(&row_ref.uid.sort))
|
||||
.flatten();
|
||||
if let Some(intval) = maybe_intval {
|
||||
acc.push(intval.to_row_val(row_ref.clone()));
|
||||
}
|
||||
}
|
||||
Ok(acc)
|
||||
}
|
||||
Selector::Prefix { shard, sort_prefix } => {
|
||||
let last_bound = prefix_last_bound(sort_prefix);
|
||||
|
||||
Ok(store
|
||||
.get(*shard)
|
||||
.unwrap_or(&BTreeMap::new())
|
||||
.range((Included(sort_prefix.to_string()), last_bound))
|
||||
.map(|(k, v)| v.to_row_val(RowRef::new(shard, k)))
|
||||
.collect::<Vec<_>>())
|
||||
}
|
||||
Selector::Single(row_ref) => {
|
||||
let intval = store
|
||||
.get(&row_ref.uid.shard)
|
||||
.ok_or(StorageError::NotFound)?
|
||||
.get(&row_ref.uid.sort)
|
||||
.ok_or(StorageError::NotFound)?;
|
||||
Ok(vec![intval.to_row_val((*row_ref).clone())])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn row_rm<'a>(&self, select: &Selector<'a>) -> Result<(), StorageError> {
|
||||
tracing::trace!(select=%select, command="row_rm");
|
||||
|
||||
let values = match select {
|
||||
Selector::Range { .. } | Selector::Prefix { .. } => self
|
||||
.row_fetch(select)
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|rv| rv.row_ref)
|
||||
.collect::<Vec<_>>(),
|
||||
Selector::List(rlist) => rlist.clone(),
|
||||
Selector::Single(row_ref) => vec![(*row_ref).clone()],
|
||||
};
|
||||
|
||||
for v in values.into_iter() {
|
||||
self.row_rm_single(&v)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn row_insert(&self, values: Vec<RowVal>) -> Result<(), StorageError> {
|
||||
tracing::trace!(entries=%values.iter().map(|v| v.row_ref.to_string()).collect::<Vec<_>>().join(","), command="row_insert");
|
||||
let mut store = self.row.write().or(Err(StorageError::Internal))?;
|
||||
for v in values.into_iter() {
|
||||
let shard = v.row_ref.uid.shard;
|
||||
let sort = v.row_ref.uid.sort;
|
||||
|
||||
let val = match v.value.into_iter().next() {
|
||||
Some(Alternative::Value(x)) => x,
|
||||
_ => vec![],
|
||||
};
|
||||
|
||||
let cauz = match v.row_ref.causality.map(|v| v.parse::<u64>()) {
|
||||
Some(Ok(v)) => v,
|
||||
_ => 0,
|
||||
};
|
||||
|
||||
let bt = store.entry(shard).or_default();
|
||||
let intval = bt.entry(sort).or_default();
|
||||
|
||||
if cauz == intval.version {
|
||||
intval.data.clear();
|
||||
}
|
||||
intval.data.push(InternalData::Value(val));
|
||||
intval.version += 1;
|
||||
intval.change.notify_waiters();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
async fn row_poll(&self, value: &RowRef) -> Result<RowVal, StorageError> {
|
||||
tracing::trace!(entry=%value, command="row_poll");
|
||||
let shard = &value.uid.shard;
|
||||
let sort = &value.uid.sort;
|
||||
let cauz = match value.causality.as_ref().map(|v| v.parse::<u64>()) {
|
||||
Some(Ok(v)) => v,
|
||||
_ => 0,
|
||||
};
|
||||
|
||||
let notify_me = {
|
||||
let mut store = self.row.write().or(Err(StorageError::Internal))?;
|
||||
let bt = store.entry(shard.to_string()).or_default();
|
||||
let intval = bt.entry(sort.to_string()).or_default();
|
||||
|
||||
if intval.version != cauz {
|
||||
return Ok(intval.to_row_val(value.clone()));
|
||||
}
|
||||
intval.change.clone()
|
||||
};
|
||||
|
||||
notify_me.notified().await;
|
||||
|
||||
let res = self.row_fetch(&Selector::Single(value)).await?;
|
||||
res.into_iter().next().ok_or(StorageError::NotFound)
|
||||
}
|
||||
|
||||
async fn blob_fetch(&self, blob_ref: &BlobRef) -> Result<BlobVal, StorageError> {
|
||||
tracing::trace!(entry=%blob_ref, command="blob_fetch");
|
||||
let store = self.blob.read().or(Err(StorageError::Internal))?;
|
||||
store
|
||||
.get(&blob_ref.0)
|
||||
.ok_or(StorageError::NotFound)
|
||||
.map(|v| v.to_blob_val(blob_ref))
|
||||
}
|
||||
async fn blob_insert(&self, blob_val: BlobVal) -> Result<(), StorageError> {
|
||||
tracing::trace!(entry=%blob_val.blob_ref, command="blob_insert");
|
||||
let mut store = self.blob.write().or(Err(StorageError::Internal))?;
|
||||
let entry = store.entry(blob_val.blob_ref.0.clone()).or_default();
|
||||
entry.data = blob_val.value.clone();
|
||||
entry.metadata = blob_val.meta.clone();
|
||||
Ok(())
|
||||
}
|
||||
async fn blob_copy(&self, src: &BlobRef, dst: &BlobRef) -> Result<(), StorageError> {
|
||||
tracing::trace!(src=%src, dst=%dst, command="blob_copy");
|
||||
let mut store = self.blob.write().or(Err(StorageError::Internal))?;
|
||||
let blob_src = store.entry(src.0.clone()).or_default().clone();
|
||||
store.insert(dst.0.clone(), blob_src);
|
||||
Ok(())
|
||||
}
|
||||
async fn blob_list(&self, prefix: &str) -> Result<Vec<BlobRef>, StorageError> {
|
||||
tracing::trace!(prefix = prefix, command = "blob_list");
|
||||
let store = self.blob.read().or(Err(StorageError::Internal))?;
|
||||
let last_bound = prefix_last_bound(prefix);
|
||||
let blist = store
|
||||
.range((Included(prefix.to_string()), last_bound))
|
||||
.map(|(k, _)| BlobRef(k.to_string()))
|
||||
.collect::<Vec<_>>();
|
||||
Ok(blist)
|
||||
}
|
||||
async fn blob_rm(&self, blob_ref: &BlobRef) -> Result<(), StorageError> {
|
||||
tracing::trace!(entry=%blob_ref, command="blob_rm");
|
||||
let mut store = self.blob.write().or(Err(StorageError::Internal))?;
|
||||
store.remove(&blob_ref.0);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
@ -1,179 +0,0 @@
|
|||
/*
|
||||
*
|
||||
* This abstraction goal is to leverage all the semantic of Garage K2V+S3,
|
||||
* to be as tailored as possible to it ; it aims to be a zero-cost abstraction
|
||||
* compared to when we where directly using the K2V+S3 client.
|
||||
*
|
||||
* My idea: we can encapsulate the causality token
|
||||
* into the object system so it is not exposed.
|
||||
*/
|
||||
|
||||
pub mod garage;
|
||||
pub mod in_memory;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use std::collections::HashMap;
|
||||
use std::hash::Hash;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum Alternative {
|
||||
Tombstone,
|
||||
Value(Vec<u8>),
|
||||
}
|
||||
type ConcurrentValues = Vec<Alternative>;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum StorageError {
|
||||
NotFound,
|
||||
Internal,
|
||||
}
|
||||
impl std::fmt::Display for StorageError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str("Storage Error: ")?;
|
||||
match self {
|
||||
Self::NotFound => f.write_str("Item not found"),
|
||||
Self::Internal => f.write_str("An internal error occured"),
|
||||
}
|
||||
}
|
||||
}
|
||||
impl std::error::Error for StorageError {}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct RowUid {
|
||||
pub shard: String,
|
||||
pub sort: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct RowRef {
|
||||
pub uid: RowUid,
|
||||
pub causality: Option<String>,
|
||||
}
|
||||
impl std::fmt::Display for RowRef {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(
|
||||
f,
|
||||
"RowRef({}, {}, {:?})",
|
||||
self.uid.shard, self.uid.sort, self.causality
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl RowRef {
|
||||
pub fn new(shard: &str, sort: &str) -> Self {
|
||||
Self {
|
||||
uid: RowUid {
|
||||
shard: shard.to_string(),
|
||||
sort: sort.to_string(),
|
||||
},
|
||||
causality: None,
|
||||
}
|
||||
}
|
||||
pub fn with_causality(mut self, causality: String) -> Self {
|
||||
self.causality = Some(causality);
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RowVal {
|
||||
pub row_ref: RowRef,
|
||||
pub value: ConcurrentValues,
|
||||
}
|
||||
|
||||
impl RowVal {
|
||||
pub fn new(row_ref: RowRef, value: Vec<u8>) -> Self {
|
||||
Self {
|
||||
row_ref,
|
||||
value: vec![Alternative::Value(value)],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BlobRef(pub String);
|
||||
impl std::fmt::Display for BlobRef {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "BlobRef({})", self.0)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BlobVal {
|
||||
pub blob_ref: BlobRef,
|
||||
pub meta: HashMap<String, String>,
|
||||
pub value: Vec<u8>,
|
||||
}
|
||||
impl BlobVal {
|
||||
pub fn new(blob_ref: BlobRef, value: Vec<u8>) -> Self {
|
||||
Self {
|
||||
blob_ref,
|
||||
value,
|
||||
meta: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_meta(mut self, k: String, v: String) -> Self {
|
||||
self.meta.insert(k, v);
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum Selector<'a> {
|
||||
Range {
|
||||
shard: &'a str,
|
||||
sort_begin: &'a str,
|
||||
sort_end: &'a str,
|
||||
},
|
||||
List(Vec<RowRef>), // list of (shard_key, sort_key)
|
||||
#[allow(dead_code)]
|
||||
Prefix {
|
||||
shard: &'a str,
|
||||
sort_prefix: &'a str,
|
||||
},
|
||||
Single(&'a RowRef),
|
||||
}
|
||||
impl<'a> std::fmt::Display for Selector<'a> {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::Range {
|
||||
shard,
|
||||
sort_begin,
|
||||
sort_end,
|
||||
} => write!(f, "Range({}, [{}, {}[)", shard, sort_begin, sort_end),
|
||||
Self::List(list) => write!(f, "List({:?})", list),
|
||||
Self::Prefix { shard, sort_prefix } => write!(f, "Prefix({}, {})", shard, sort_prefix),
|
||||
Self::Single(row_ref) => write!(f, "Single({})", row_ref),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
pub trait IStore {
|
||||
async fn row_fetch<'a>(&self, select: &Selector<'a>) -> Result<Vec<RowVal>, StorageError>;
|
||||
async fn row_rm<'a>(&self, select: &Selector<'a>) -> Result<(), StorageError>;
|
||||
async fn row_insert(&self, values: Vec<RowVal>) -> Result<(), StorageError>;
|
||||
async fn row_poll(&self, value: &RowRef) -> Result<RowVal, StorageError>;
|
||||
|
||||
async fn blob_fetch(&self, blob_ref: &BlobRef) -> Result<BlobVal, StorageError>;
|
||||
async fn blob_insert(&self, blob_val: BlobVal) -> Result<(), StorageError>;
|
||||
async fn blob_copy(&self, src: &BlobRef, dst: &BlobRef) -> Result<(), StorageError>;
|
||||
async fn blob_list(&self, prefix: &str) -> Result<Vec<BlobRef>, StorageError>;
|
||||
async fn blob_rm(&self, blob_ref: &BlobRef) -> Result<(), StorageError>;
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
|
||||
pub struct UnicityBuffer(Vec<u8>);
|
||||
|
||||
#[async_trait]
|
||||
pub trait IBuilder: std::fmt::Debug {
|
||||
async fn build(&self) -> Result<Store, StorageError>;
|
||||
|
||||
/// Returns an opaque buffer that uniquely identifies this builder
|
||||
fn unique(&self) -> UnicityBuffer;
|
||||
}
|
||||
|
||||
pub type Builder = Arc<dyn IBuilder + Send + Sync>;
|
||||
pub type Store = Box<dyn IStore + Send + Sync>;
|
||||
9
src/time.rs
Normal file
9
src/time.rs
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
/// Returns milliseconds since UNIX Epoch
|
||||
pub fn now_msec() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.expect("Fix your clock :o")
|
||||
.as_millis() as u64
|
||||
}
|
||||
|
|
@ -1,65 +0,0 @@
|
|||
use rand::prelude::*;
|
||||
use std::str::FromStr;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
/// Returns milliseconds since UNIX Epoch
|
||||
pub fn now_msec() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.expect("Fix your clock :o")
|
||||
.as_millis() as u64
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)]
|
||||
pub struct Timestamp {
|
||||
pub msec: u64,
|
||||
pub rand: u64,
|
||||
}
|
||||
|
||||
impl Timestamp {
|
||||
#[allow(dead_code)]
|
||||
// 2023-05-15 try to make clippy happy and not sure if this fn will be used in the future.
|
||||
pub fn now() -> Self {
|
||||
let mut rng = thread_rng();
|
||||
Self {
|
||||
msec: now_msec(),
|
||||
rand: rng.gen::<u64>(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn after(other: &Self) -> Self {
|
||||
let mut rng = thread_rng();
|
||||
Self {
|
||||
msec: std::cmp::max(now_msec(), other.msec + 1),
|
||||
rand: rng.gen::<u64>(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn zero() -> Self {
|
||||
Self { msec: 0, rand: 0 }
|
||||
}
|
||||
}
|
||||
|
||||
impl ToString for Timestamp {
|
||||
fn to_string(&self) -> String {
|
||||
let mut bytes = [0u8; 16];
|
||||
bytes[0..8].copy_from_slice(&u64::to_be_bytes(self.msec));
|
||||
bytes[8..16].copy_from_slice(&u64::to_be_bytes(self.rand));
|
||||
hex::encode(bytes)
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for Timestamp {
|
||||
type Err = &'static str;
|
||||
|
||||
fn from_str(s: &str) -> Result<Timestamp, &'static str> {
|
||||
let bytes = hex::decode(s).map_err(|_| "invalid hex")?;
|
||||
if bytes.len() != 16 {
|
||||
return Err("bad length");
|
||||
}
|
||||
Ok(Self {
|
||||
msec: u64::from_be_bytes(bytes[0..8].try_into().unwrap()),
|
||||
rand: u64::from_be_bytes(bytes[8..16].try_into().unwrap()),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -1,357 +0,0 @@
|
|||
use anyhow::Context;
|
||||
|
||||
mod common;
|
||||
use crate::common::constants::*;
|
||||
use crate::common::fragments::*;
|
||||
|
||||
fn main() {
|
||||
rfc3501_imap4rev1_base();
|
||||
rfc6851_imapext_move();
|
||||
rfc4551_imapext_condstore();
|
||||
rfc2177_imapext_idle();
|
||||
rfc5161_imapext_enable(); // 1
|
||||
rfc3691_imapext_unselect(); // 2
|
||||
rfc7888_imapext_literal(); // 3
|
||||
rfc4315_imapext_uidplus(); // 4
|
||||
rfc5819_imapext_liststatus(); // 5
|
||||
println!("✅ SUCCESS 🌟🚀🥳🙏🥹");
|
||||
}
|
||||
|
||||
fn rfc3501_imap4rev1_base() {
|
||||
println!("🧪 rfc3501_imap4rev1_base");
|
||||
common::aerogramme_provider_daemon_dev(|imap_socket, lmtp_socket| {
|
||||
connect(imap_socket).context("server says hello")?;
|
||||
capability(imap_socket, Extension::None).context("check server capabilities")?;
|
||||
login(imap_socket, Account::Alice).context("login test")?;
|
||||
create_mailbox(imap_socket, Mailbox::Archive).context("created mailbox archive")?;
|
||||
let select_res =
|
||||
select(imap_socket, Mailbox::Inbox, SelectMod::None).context("select inbox")?;
|
||||
assert!(select_res.contains("* 0 EXISTS"));
|
||||
|
||||
check(imap_socket).context("check must run")?;
|
||||
status(imap_socket, Mailbox::Archive, StatusKind::UidNext)
|
||||
.context("status of archive from inbox")?;
|
||||
lmtp_handshake(lmtp_socket).context("handshake lmtp done")?;
|
||||
lmtp_deliver_email(lmtp_socket, Email::Multipart).context("mail delivered successfully")?;
|
||||
noop_exists(imap_socket, 1).context("noop loop must detect a new email")?;
|
||||
|
||||
let srv_msg = fetch(
|
||||
imap_socket,
|
||||
Selection::FirstId,
|
||||
FetchKind::Rfc822,
|
||||
FetchMod::None,
|
||||
)
|
||||
.context("fetch rfc822 message, should be our first message")?;
|
||||
let orig_email = std::str::from_utf8(EMAIL1)?;
|
||||
assert!(srv_msg.contains(orig_email));
|
||||
|
||||
copy(imap_socket, Selection::FirstId, Mailbox::Archive)
|
||||
.context("copy message to the archive mailbox")?;
|
||||
append(imap_socket, Email::Basic).context("insert email in INBOX")?;
|
||||
noop_exists(imap_socket, 2).context("noop loop must detect a new email")?;
|
||||
search(imap_socket, SearchKind::Text("OoOoO")).expect("search should return something");
|
||||
store(
|
||||
imap_socket,
|
||||
Selection::FirstId,
|
||||
Flag::Deleted,
|
||||
StoreAction::AddFlags,
|
||||
StoreMod::None,
|
||||
)
|
||||
.context("should add delete flag to the email")?;
|
||||
expunge(imap_socket).context("expunge emails")?;
|
||||
rename_mailbox(imap_socket, Mailbox::Archive, Mailbox::Drafts)
|
||||
.context("Archive mailbox is renamed Drafts")?;
|
||||
delete_mailbox(imap_socket, Mailbox::Drafts).context("Drafts mailbox is deleted")?;
|
||||
Ok(())
|
||||
})
|
||||
.expect("test fully run");
|
||||
}
|
||||
|
||||
fn rfc3691_imapext_unselect() {
|
||||
println!("🧪 rfc3691_imapext_unselect");
|
||||
common::aerogramme_provider_daemon_dev(|imap_socket, lmtp_socket| {
|
||||
connect(imap_socket).context("server says hello")?;
|
||||
|
||||
lmtp_handshake(lmtp_socket).context("handshake lmtp done")?;
|
||||
lmtp_deliver_email(lmtp_socket, Email::Basic).context("mail delivered successfully")?;
|
||||
|
||||
capability(imap_socket, Extension::Unselect).context("check server capabilities")?;
|
||||
login(imap_socket, Account::Alice).context("login test")?;
|
||||
let select_res =
|
||||
select(imap_socket, Mailbox::Inbox, SelectMod::None).context("select inbox")?;
|
||||
assert!(select_res.contains("* 0 EXISTS"));
|
||||
|
||||
noop_exists(imap_socket, 1).context("noop loop must detect a new email")?;
|
||||
store(
|
||||
imap_socket,
|
||||
Selection::FirstId,
|
||||
Flag::Deleted,
|
||||
StoreAction::AddFlags,
|
||||
StoreMod::None,
|
||||
)
|
||||
.context("add delete flags to the email")?;
|
||||
unselect(imap_socket)
|
||||
.context("unselect inbox while preserving email with the \\Delete flag")?;
|
||||
let select_res =
|
||||
select(imap_socket, Mailbox::Inbox, SelectMod::None).context("select inbox again")?;
|
||||
assert!(select_res.contains("* 1 EXISTS"));
|
||||
|
||||
let srv_msg = fetch(
|
||||
imap_socket,
|
||||
Selection::FirstId,
|
||||
FetchKind::Rfc822,
|
||||
FetchMod::None,
|
||||
)
|
||||
.context("message is still present")?;
|
||||
let orig_email = std::str::from_utf8(EMAIL2)?;
|
||||
assert!(srv_msg.contains(orig_email));
|
||||
|
||||
close(imap_socket).context("close inbox and expunge message")?;
|
||||
let select_res = select(imap_socket, Mailbox::Inbox, SelectMod::None)
|
||||
.context("select inbox again and check it's empty")?;
|
||||
assert!(select_res.contains("* 0 EXISTS"));
|
||||
|
||||
Ok(())
|
||||
})
|
||||
.expect("test fully run");
|
||||
}
|
||||
|
||||
fn rfc5161_imapext_enable() {
|
||||
println!("🧪 rfc5161_imapext_enable");
|
||||
common::aerogramme_provider_daemon_dev(|imap_socket, _lmtp_socket| {
|
||||
connect(imap_socket).context("server says hello")?;
|
||||
login(imap_socket, Account::Alice).context("login test")?;
|
||||
enable(imap_socket, Enable::Utf8Accept, Some(Enable::Utf8Accept))?;
|
||||
enable(imap_socket, Enable::Utf8Accept, None)?;
|
||||
logout(imap_socket)?;
|
||||
|
||||
Ok(())
|
||||
})
|
||||
.expect("test fully run");
|
||||
}
|
||||
|
||||
fn rfc6851_imapext_move() {
|
||||
println!("🧪 rfc6851_imapext_move");
|
||||
common::aerogramme_provider_daemon_dev(|imap_socket, lmtp_socket| {
|
||||
connect(imap_socket).context("server says hello")?;
|
||||
|
||||
capability(imap_socket, Extension::Move).context("check server capabilities")?;
|
||||
login(imap_socket, Account::Alice).context("login test")?;
|
||||
create_mailbox(imap_socket, Mailbox::Archive).context("created mailbox archive")?;
|
||||
let select_res =
|
||||
select(imap_socket, Mailbox::Inbox, SelectMod::None).context("select inbox")?;
|
||||
assert!(select_res.contains("* 0 EXISTS"));
|
||||
|
||||
lmtp_handshake(lmtp_socket).context("handshake lmtp done")?;
|
||||
lmtp_deliver_email(lmtp_socket, Email::Basic).context("mail delivered successfully")?;
|
||||
|
||||
noop_exists(imap_socket, 1).context("noop loop must detect a new email")?;
|
||||
r#move(imap_socket, Selection::FirstId, Mailbox::Archive)
|
||||
.context("message from inbox moved to archive")?;
|
||||
|
||||
unselect(imap_socket)
|
||||
.context("unselect inbox while preserving email with the \\Delete flag")?;
|
||||
let select_res =
|
||||
select(imap_socket, Mailbox::Archive, SelectMod::None).context("select archive")?;
|
||||
assert!(select_res.contains("* 1 EXISTS"));
|
||||
|
||||
let srv_msg = fetch(
|
||||
imap_socket,
|
||||
Selection::FirstId,
|
||||
FetchKind::Rfc822,
|
||||
FetchMod::None,
|
||||
)
|
||||
.context("check mail exists")?;
|
||||
let orig_email = std::str::from_utf8(EMAIL2)?;
|
||||
assert!(srv_msg.contains(orig_email));
|
||||
|
||||
logout(imap_socket).context("must quit")?;
|
||||
|
||||
Ok(())
|
||||
})
|
||||
.expect("test fully run");
|
||||
}
|
||||
|
||||
fn rfc7888_imapext_literal() {
|
||||
println!("🧪 rfc7888_imapext_literal");
|
||||
common::aerogramme_provider_daemon_dev(|imap_socket, _lmtp_socket| {
|
||||
connect(imap_socket).context("server says hello")?;
|
||||
|
||||
capability(imap_socket, Extension::LiteralPlus).context("check server capabilities")?;
|
||||
login_with_literal(imap_socket, Account::Alice).context("use literal to connect Alice")?;
|
||||
|
||||
Ok(())
|
||||
})
|
||||
.expect("test fully run");
|
||||
}
|
||||
|
||||
fn rfc4551_imapext_condstore() {
|
||||
println!("🧪 rfc4551_imapext_condstore");
|
||||
common::aerogramme_provider_daemon_dev(|imap_socket, lmtp_socket| {
|
||||
// Setup the test
|
||||
connect(imap_socket).context("server says hello")?;
|
||||
|
||||
// RFC 3.1.1 Advertising Support for CONDSTORE
|
||||
capability(imap_socket, Extension::Condstore).context("check server capabilities")?;
|
||||
login(imap_socket, Account::Alice).context("login test")?;
|
||||
|
||||
// RFC 3.1.8. CONDSTORE Parameter to SELECT and EXAMINE
|
||||
let select_res =
|
||||
select(imap_socket, Mailbox::Inbox, SelectMod::Condstore).context("select inbox")?;
|
||||
// RFC 3.1.2 New OK Untagged Responses for SELECT and EXAMINE
|
||||
assert!(select_res.contains("[HIGHESTMODSEQ 1]"));
|
||||
|
||||
// RFC 3.1.3. STORE and UID STORE Commands
|
||||
lmtp_handshake(lmtp_socket).context("handshake lmtp done")?;
|
||||
lmtp_deliver_email(lmtp_socket, Email::Basic).context("mail delivered successfully")?;
|
||||
lmtp_deliver_email(lmtp_socket, Email::Multipart).context("mail delivered successfully")?;
|
||||
noop_exists(imap_socket, 2).context("noop loop must detect a new email")?;
|
||||
let store_res = store(
|
||||
imap_socket,
|
||||
Selection::All,
|
||||
Flag::Important,
|
||||
StoreAction::AddFlags,
|
||||
StoreMod::UnchangedSince(1),
|
||||
)?;
|
||||
assert!(store_res.contains("[MODIFIED 2]"));
|
||||
assert!(store_res.contains("* 1 FETCH (FLAGS (\\Important) MODSEQ (3))"));
|
||||
assert!(!store_res.contains("* 2 FETCH"));
|
||||
assert_eq!(store_res.lines().count(), 2);
|
||||
|
||||
// RFC 3.1.4. FETCH and UID FETCH Commands
|
||||
let fetch_res = fetch(
|
||||
imap_socket,
|
||||
Selection::All,
|
||||
FetchKind::Rfc822Size,
|
||||
FetchMod::ChangedSince(2),
|
||||
)?;
|
||||
assert!(fetch_res.contains("* 1 FETCH (RFC822.SIZE 81 MODSEQ (3))"));
|
||||
assert!(!fetch_res.contains("* 2 FETCH"));
|
||||
assert_eq!(store_res.lines().count(), 2);
|
||||
|
||||
// RFC 3.1.5. MODSEQ Search Criterion in SEARCH
|
||||
let search_res = search(imap_socket, SearchKind::ModSeq(3))?;
|
||||
// RFC 3.1.6. Modified SEARCH Untagged Response
|
||||
assert!(search_res.contains("* SEARCH 1 (MODSEQ 3)"));
|
||||
|
||||
// RFC 3.1.7 HIGHESTMODSEQ Status Data Items
|
||||
let status_res = status(imap_socket, Mailbox::Inbox, StatusKind::HighestModSeq)?;
|
||||
assert!(status_res.contains("HIGHESTMODSEQ 3"));
|
||||
|
||||
Ok(())
|
||||
})
|
||||
.expect("test fully run");
|
||||
}
|
||||
|
||||
fn rfc2177_imapext_idle() {
|
||||
println!("🧪 rfc2177_imapext_idle");
|
||||
common::aerogramme_provider_daemon_dev(|imap_socket, lmtp_socket| {
|
||||
// Test setup, check capability
|
||||
connect(imap_socket).context("server says hello")?;
|
||||
capability(imap_socket, Extension::Idle).context("check server capabilities")?;
|
||||
login(imap_socket, Account::Alice).context("login test")?;
|
||||
select(imap_socket, Mailbox::Inbox, SelectMod::None).context("select inbox")?;
|
||||
|
||||
// Check that new messages from LMTP are correctly detected during idling
|
||||
start_idle(imap_socket).context("can't start idling")?;
|
||||
lmtp_handshake(lmtp_socket).context("handshake lmtp done")?;
|
||||
lmtp_deliver_email(lmtp_socket, Email::Basic).context("mail delivered successfully")?;
|
||||
let srv_msg = stop_idle(imap_socket).context("stop idling")?;
|
||||
assert!(srv_msg.contains("* 1 EXISTS"));
|
||||
|
||||
Ok(())
|
||||
})
|
||||
.expect("test fully run");
|
||||
}
|
||||
|
||||
fn rfc4315_imapext_uidplus() {
|
||||
println!("🧪 rfc4315_imapext_uidplus");
|
||||
common::aerogramme_provider_daemon_dev(|imap_socket, lmtp_socket| {
|
||||
// Test setup, check capability, insert 2 emails
|
||||
connect(imap_socket).context("server says hello")?;
|
||||
capability(imap_socket, Extension::UidPlus).context("check server capabilities")?;
|
||||
login(imap_socket, Account::Alice).context("login test")?;
|
||||
select(imap_socket, Mailbox::Inbox, SelectMod::None).context("select inbox")?;
|
||||
lmtp_handshake(lmtp_socket).context("handshake lmtp done")?;
|
||||
lmtp_deliver_email(lmtp_socket, Email::Basic).context("mail delivered successfully")?;
|
||||
lmtp_deliver_email(lmtp_socket, Email::Multipart).context("mail delivered successfully")?;
|
||||
noop_exists(imap_socket, 2).context("noop loop must detect a new email")?;
|
||||
|
||||
// Check UID EXPUNGE seqset
|
||||
store(
|
||||
imap_socket,
|
||||
Selection::All,
|
||||
Flag::Deleted,
|
||||
StoreAction::AddFlags,
|
||||
StoreMod::None,
|
||||
)?;
|
||||
let res = uid_expunge(imap_socket, Selection::FirstId)?;
|
||||
assert_eq!(res.lines().count(), 2);
|
||||
assert!(res.contains("* 1 EXPUNGE"));
|
||||
|
||||
// APPENDUID check UID + UID VALIDITY
|
||||
// Note: 4 and not 3, as we update the UID counter when we delete an email
|
||||
// it's part of our UID proof
|
||||
let res = append(imap_socket, Email::Multipart)?;
|
||||
assert!(res.contains("[APPENDUID 1 4]"));
|
||||
|
||||
// COPYUID, check
|
||||
create_mailbox(imap_socket, Mailbox::Archive).context("created mailbox archive")?;
|
||||
let res = copy(imap_socket, Selection::FirstId, Mailbox::Archive)?;
|
||||
assert!(res.contains("[COPYUID 1 2 1]"));
|
||||
|
||||
// MOVEUID, check
|
||||
let res = r#move(imap_socket, Selection::FirstId, Mailbox::Archive)?;
|
||||
assert!(res.contains("[COPYUID 1 2 2]"));
|
||||
|
||||
Ok(())
|
||||
})
|
||||
.expect("test fully run");
|
||||
}
|
||||
|
||||
///
|
||||
/// Example
|
||||
///
|
||||
/// ```text
|
||||
/// 30 list "" "*" RETURN (STATUS (MESSAGES UNSEEN))
|
||||
/// * LIST (\Subscribed) "." INBOX
|
||||
/// * STATUS INBOX (MESSAGES 2 UNSEEN 1)
|
||||
/// 30 OK LIST completed
|
||||
/// ```
|
||||
fn rfc5819_imapext_liststatus() {
|
||||
println!("🧪 rfc5819_imapext_liststatus");
|
||||
common::aerogramme_provider_daemon_dev(|imap_socket, lmtp_socket| {
|
||||
// Test setup, check capability, add 2 emails, read 1
|
||||
connect(imap_socket).context("server says hello")?;
|
||||
capability(imap_socket, Extension::ListStatus).context("check server capabilities")?;
|
||||
login(imap_socket, Account::Alice).context("login test")?;
|
||||
select(imap_socket, Mailbox::Inbox, SelectMod::None).context("select inbox")?;
|
||||
lmtp_handshake(lmtp_socket).context("handshake lmtp done")?;
|
||||
lmtp_deliver_email(lmtp_socket, Email::Basic).context("mail delivered successfully")?;
|
||||
lmtp_deliver_email(lmtp_socket, Email::Multipart).context("mail delivered successfully")?;
|
||||
noop_exists(imap_socket, 2).context("noop loop must detect a new email")?;
|
||||
fetch(
|
||||
imap_socket,
|
||||
Selection::FirstId,
|
||||
FetchKind::Rfc822,
|
||||
FetchMod::None,
|
||||
)
|
||||
.context("read one message")?;
|
||||
close(imap_socket).context("close inbox")?;
|
||||
|
||||
// Test return status MESSAGES UNSEEN
|
||||
let ret = list(
|
||||
imap_socket,
|
||||
MbxSelect::All,
|
||||
ListReturn::StatusMessagesUnseen,
|
||||
)?;
|
||||
assert!(ret.contains("* STATUS INBOX (MESSAGES 2 UNSEEN 1)"));
|
||||
|
||||
// Test that without RETURN, no status is sent
|
||||
let ret = list(imap_socket, MbxSelect::All, ListReturn::None)?;
|
||||
assert!(!ret.contains("* STATUS"));
|
||||
|
||||
Ok(())
|
||||
})
|
||||
.expect("test fully run");
|
||||
}
|
||||
|
|
@ -1,54 +0,0 @@
|
|||
use std::time;
|
||||
|
||||
pub static SMALL_DELAY: time::Duration = time::Duration::from_millis(200);
|
||||
|
||||
pub static EMAIL1: &[u8] = b"Date: Sat, 8 Jul 2023 07:14:29 +0200\r
|
||||
From: Bob Robert <bob@example.tld>\r
|
||||
To: Alice Malice <alice@example.tld>\r
|
||||
CC: =?ISO-8859-1?Q?Andr=E9?= Pirard <PIRARD@vm1.ulg.ac.be>\r
|
||||
Subject: =?ISO-8859-1?B?SWYgeW91IGNhbiByZWFkIHRoaXMgeW8=?=\r
|
||||
=?ISO-8859-2?B?dSB1bmRlcnN0YW5kIHRoZSBleGFtcGxlLg==?=\r
|
||||
X-Unknown: something something\r
|
||||
Bad entry\r
|
||||
on multiple lines\r
|
||||
Message-ID: <NTAxNzA2AC47634Y366BAMTY4ODc5MzQyODY0ODY5@www.grrrndzero.org>\r
|
||||
MIME-Version: 1.0\r
|
||||
Content-Type: multipart/alternative;\r
|
||||
boundary=\"b1_e376dc71bafc953c0b0fdeb9983a9956\"\r
|
||||
Content-Transfer-Encoding: 7bit\r
|
||||
\r
|
||||
This is a multi-part message in MIME format.\r
|
||||
\r
|
||||
--b1_e376dc71bafc953c0b0fdeb9983a9956\r
|
||||
Content-Type: text/plain; charset=utf-8\r
|
||||
Content-Transfer-Encoding: quoted-printable\r
|
||||
\r
|
||||
GZ\r
|
||||
OoOoO\r
|
||||
oOoOoOoOo\r
|
||||
oOoOoOoOoOoOoOoOo\r
|
||||
oOoOoOoOoOoOoOoOoOoOoOo\r
|
||||
oOoOoOoOoOoOoOoOoOoOoOoOoOoOo\r
|
||||
OoOoOoOoOoOoOoOoOoOoOoOoOoOoOoOoO\r
|
||||
\r
|
||||
--b1_e376dc71bafc953c0b0fdeb9983a9956\r
|
||||
Content-Type: text/html; charset=us-ascii\r
|
||||
\r
|
||||
<div style=\"text-align: center;\"><strong>GZ</strong><br />\r
|
||||
OoOoO<br />\r
|
||||
oOoOoOoOo<br />\r
|
||||
oOoOoOoOoOoOoOoOo<br />\r
|
||||
oOoOoOoOoOoOoOoOoOoOoOo<br />\r
|
||||
oOoOoOoOoOoOoOoOoOoOoOoOoOoOo<br />\r
|
||||
OoOoOoOoOoOoOoOoOoOoOoOoOoOoOoOoO<br />\r
|
||||
</div>\r
|
||||
\r
|
||||
--b1_e376dc71bafc953c0b0fdeb9983a9956--\r
|
||||
";
|
||||
|
||||
pub static EMAIL2: &[u8] = b"From: alice@example.com\r
|
||||
To: alice@example.tld\r
|
||||
Subject: Test\r
|
||||
\r
|
||||
Hello world!\r
|
||||
";
|
||||
|
|
@ -1,570 +0,0 @@
|
|||
use anyhow::{bail, Result};
|
||||
use std::io::Write;
|
||||
use std::net::TcpStream;
|
||||
use std::thread;
|
||||
|
||||
use crate::common::constants::*;
|
||||
use crate::common::*;
|
||||
|
||||
/// These fragments are not a generic IMAP client
|
||||
/// but specialized to our specific tests. They can't take
|
||||
/// arbitrary values, only enum for which the code is known
|
||||
/// to be correct. The idea is that the generated message is more
|
||||
/// or less hardcoded by the developer, so its clear what's expected,
|
||||
/// and not generated by a library. Also don't use vector of enum,
|
||||
/// as it again introduce some kind of genericity we try so hard to avoid:
|
||||
/// instead add a dedicated enum, for example "All" or anything relaevent that would
|
||||
/// describe your list and then hardcode it in your fragment.
|
||||
/// DON'T. TRY. TO. BE. GENERIC. HERE.
|
||||
|
||||
pub fn connect(imap: &mut TcpStream) -> Result<()> {
|
||||
let mut buffer: [u8; 1500] = [0; 1500];
|
||||
|
||||
let read = read_lines(imap, &mut buffer, None)?;
|
||||
assert_eq!(&read[..4], &b"* OK"[..]);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub enum Account {
|
||||
Alice,
|
||||
}
|
||||
|
||||
pub enum Extension {
|
||||
None,
|
||||
Unselect,
|
||||
Move,
|
||||
Condstore,
|
||||
LiteralPlus,
|
||||
Idle,
|
||||
UidPlus,
|
||||
ListStatus,
|
||||
}
|
||||
|
||||
pub enum Enable {
|
||||
Utf8Accept,
|
||||
CondStore,
|
||||
All,
|
||||
}
|
||||
|
||||
pub enum Mailbox {
|
||||
Inbox,
|
||||
Archive,
|
||||
Drafts,
|
||||
}
|
||||
|
||||
pub enum Flag {
|
||||
Deleted,
|
||||
Important,
|
||||
}
|
||||
|
||||
pub enum Email {
|
||||
Basic,
|
||||
Multipart,
|
||||
}
|
||||
|
||||
pub enum Selection {
|
||||
FirstId,
|
||||
SecondId,
|
||||
All,
|
||||
}
|
||||
|
||||
pub enum SelectMod {
|
||||
None,
|
||||
Condstore,
|
||||
}
|
||||
|
||||
pub enum StoreAction {
|
||||
AddFlags,
|
||||
DelFlags,
|
||||
SetFlags,
|
||||
AddFlagsSilent,
|
||||
DelFlagsSilent,
|
||||
SetFlagsSilent,
|
||||
}
|
||||
|
||||
pub enum StoreMod {
|
||||
None,
|
||||
UnchangedSince(u64),
|
||||
}
|
||||
|
||||
pub enum FetchKind {
|
||||
Rfc822,
|
||||
Rfc822Size,
|
||||
}
|
||||
|
||||
pub enum FetchMod {
|
||||
None,
|
||||
ChangedSince(u64),
|
||||
}
|
||||
|
||||
pub enum SearchKind<'a> {
|
||||
Text(&'a str),
|
||||
ModSeq(u64),
|
||||
}
|
||||
|
||||
pub enum StatusKind {
|
||||
UidNext,
|
||||
HighestModSeq,
|
||||
}
|
||||
|
||||
pub enum MbxSelect {
|
||||
All,
|
||||
}
|
||||
|
||||
pub enum ListReturn {
|
||||
None,
|
||||
StatusMessagesUnseen,
|
||||
}
|
||||
|
||||
pub fn capability(imap: &mut TcpStream, ext: Extension) -> Result<()> {
|
||||
imap.write(&b"5 capability\r\n"[..])?;
|
||||
|
||||
let maybe_ext = match ext {
|
||||
Extension::None => None,
|
||||
Extension::Unselect => Some("UNSELECT"),
|
||||
Extension::Move => Some("MOVE"),
|
||||
Extension::Condstore => Some("CONDSTORE"),
|
||||
Extension::LiteralPlus => Some("LITERAL+"),
|
||||
Extension::Idle => Some("IDLE"),
|
||||
Extension::UidPlus => Some("UIDPLUS"),
|
||||
Extension::ListStatus => Some("LIST-STATUS"),
|
||||
};
|
||||
|
||||
let mut buffer: [u8; 6000] = [0; 6000];
|
||||
let read = read_lines(imap, &mut buffer, Some(&b"5 OK"[..]))?;
|
||||
let srv_msg = std::str::from_utf8(read)?;
|
||||
assert!(srv_msg.contains("IMAP4REV1"));
|
||||
if let Some(ext) = maybe_ext {
|
||||
assert!(srv_msg.contains(ext));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn login(imap: &mut TcpStream, account: Account) -> Result<()> {
|
||||
let mut buffer: [u8; 1500] = [0; 1500];
|
||||
|
||||
assert!(matches!(account, Account::Alice));
|
||||
imap.write(&b"10 login alice hunter2\r\n"[..])?;
|
||||
|
||||
let read = read_lines(imap, &mut buffer, None)?;
|
||||
assert_eq!(&read[..5], &b"10 OK"[..]);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn login_with_literal(imap: &mut TcpStream, account: Account) -> Result<()> {
|
||||
let mut buffer: [u8; 1500] = [0; 1500];
|
||||
|
||||
assert!(matches!(account, Account::Alice));
|
||||
imap.write(&b"10 login {5+}\r\nalice {7+}\r\nhunter2\r\n"[..])?;
|
||||
let _read = read_lines(imap, &mut buffer, Some(&b"10 OK"[..]))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn create_mailbox(imap: &mut TcpStream, mbx: Mailbox) -> Result<()> {
|
||||
let mut buffer: [u8; 1500] = [0; 1500];
|
||||
|
||||
let mbx_str = match mbx {
|
||||
Mailbox::Inbox => "INBOX",
|
||||
Mailbox::Archive => "ArchiveCustom",
|
||||
Mailbox::Drafts => "DraftsCustom",
|
||||
};
|
||||
|
||||
let cmd = format!("15 create {}\r\n", mbx_str);
|
||||
imap.write(cmd.as_bytes())?;
|
||||
let read = read_lines(imap, &mut buffer, None)?;
|
||||
assert_eq!(&read[..12], &b"15 OK CREATE"[..]);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn list(imap: &mut TcpStream, select: MbxSelect, mod_return: ListReturn) -> Result<String> {
|
||||
let mut buffer: [u8; 6000] = [0; 6000];
|
||||
|
||||
let select_str = match select {
|
||||
MbxSelect::All => "%",
|
||||
};
|
||||
|
||||
let mod_return_str = match mod_return {
|
||||
ListReturn::None => "",
|
||||
ListReturn::StatusMessagesUnseen => " RETURN (STATUS (MESSAGES UNSEEN))",
|
||||
};
|
||||
|
||||
imap.write(format!("19 LIST \"\" \"{}\"{}\r\n", select_str, mod_return_str).as_bytes())?;
|
||||
|
||||
let read = read_lines(imap, &mut buffer, Some(&b"19 OK"[..]))?;
|
||||
let srv_msg = std::str::from_utf8(read)?;
|
||||
Ok(srv_msg.to_string())
|
||||
}
|
||||
|
||||
pub fn select(imap: &mut TcpStream, mbx: Mailbox, modifier: SelectMod) -> Result<String> {
|
||||
let mut buffer: [u8; 6000] = [0; 6000];
|
||||
|
||||
let mbx_str = match mbx {
|
||||
Mailbox::Inbox => "INBOX",
|
||||
Mailbox::Archive => "ArchiveCustom",
|
||||
Mailbox::Drafts => "DraftsCustom",
|
||||
};
|
||||
|
||||
let mod_str = match modifier {
|
||||
SelectMod::Condstore => " (CONDSTORE)",
|
||||
SelectMod::None => "",
|
||||
};
|
||||
|
||||
imap.write(format!("20 select {}{}\r\n", mbx_str, mod_str).as_bytes())?;
|
||||
|
||||
let read = read_lines(imap, &mut buffer, Some(&b"20 OK"[..]))?;
|
||||
let srv_msg = std::str::from_utf8(read)?;
|
||||
|
||||
Ok(srv_msg.to_string())
|
||||
}
|
||||
|
||||
pub fn unselect(imap: &mut TcpStream) -> Result<()> {
|
||||
imap.write(&b"70 unselect\r\n"[..])?;
|
||||
let mut buffer: [u8; 1500] = [0; 1500];
|
||||
let _read = read_lines(imap, &mut buffer, Some(&b"70 OK"[..]))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn check(imap: &mut TcpStream) -> Result<()> {
|
||||
let mut buffer: [u8; 1500] = [0; 1500];
|
||||
|
||||
imap.write(&b"21 check\r\n"[..])?;
|
||||
let _read = read_lines(imap, &mut buffer, Some(&b"21 OK"[..]))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn status(imap: &mut TcpStream, mbx: Mailbox, sk: StatusKind) -> Result<String> {
|
||||
let mbx_str = match mbx {
|
||||
Mailbox::Inbox => "INBOX",
|
||||
Mailbox::Archive => "ArchiveCustom",
|
||||
Mailbox::Drafts => "DraftsCustom",
|
||||
};
|
||||
let sk_str = match sk {
|
||||
StatusKind::UidNext => "(UIDNEXT)",
|
||||
StatusKind::HighestModSeq => "(HIGHESTMODSEQ)",
|
||||
};
|
||||
imap.write(format!("25 STATUS {} {}\r\n", mbx_str, sk_str).as_bytes())?;
|
||||
let mut buffer: [u8; 6000] = [0; 6000];
|
||||
let read = read_lines(imap, &mut buffer, Some(&b"25 OK"[..]))?;
|
||||
let srv_msg = std::str::from_utf8(read)?;
|
||||
|
||||
Ok(srv_msg.to_string())
|
||||
}
|
||||
|
||||
pub fn lmtp_handshake(lmtp: &mut TcpStream) -> Result<()> {
|
||||
let mut buffer: [u8; 1500] = [0; 1500];
|
||||
|
||||
let _read = read_lines(lmtp, &mut buffer, None)?;
|
||||
assert_eq!(&buffer[..4], &b"220 "[..]);
|
||||
|
||||
lmtp.write(&b"LHLO example.tld\r\n"[..])?;
|
||||
let _read = read_lines(lmtp, &mut buffer, Some(&b"250 "[..]))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn lmtp_deliver_email(lmtp: &mut TcpStream, email_type: Email) -> Result<()> {
|
||||
let mut buffer: [u8; 1500] = [0; 1500];
|
||||
|
||||
let email = match email_type {
|
||||
Email::Basic => EMAIL2,
|
||||
Email::Multipart => EMAIL1,
|
||||
};
|
||||
lmtp.write(&b"MAIL FROM:<bob@example.tld>\r\n"[..])?;
|
||||
let _read = read_lines(lmtp, &mut buffer, Some(&b"250 2.0.0"[..]))?;
|
||||
|
||||
lmtp.write(&b"RCPT TO:<alice@example.tld>\r\n"[..])?;
|
||||
let _read = read_lines(lmtp, &mut buffer, Some(&b"250 2.1.5"[..]))?;
|
||||
|
||||
lmtp.write(&b"DATA\r\n"[..])?;
|
||||
let _read = read_lines(lmtp, &mut buffer, Some(&b"354 "[..]))?;
|
||||
|
||||
lmtp.write(email)?;
|
||||
lmtp.write(&b"\r\n.\r\n"[..])?;
|
||||
let _read = read_lines(lmtp, &mut buffer, Some(&b"250 2.0.0"[..]))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn noop_exists(imap: &mut TcpStream, must_exists: u32) -> Result<()> {
|
||||
let mut buffer: [u8; 6000] = [0; 6000];
|
||||
|
||||
let mut max_retry = 20;
|
||||
loop {
|
||||
max_retry -= 1;
|
||||
imap.write(&b"30 NOOP\r\n"[..])?;
|
||||
let read = read_lines(imap, &mut buffer, Some(&b"30 OK"[..]))?;
|
||||
let srv_msg = std::str::from_utf8(read)?;
|
||||
|
||||
for line in srv_msg.lines() {
|
||||
if line.contains("EXISTS") {
|
||||
let got = read_first_u32(line)?;
|
||||
if got == must_exists {
|
||||
// Done
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if max_retry <= 0 {
|
||||
// Failed
|
||||
bail!("no more retry");
|
||||
}
|
||||
|
||||
thread::sleep(SMALL_DELAY);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn fetch(
|
||||
imap: &mut TcpStream,
|
||||
selection: Selection,
|
||||
kind: FetchKind,
|
||||
modifier: FetchMod,
|
||||
) -> Result<String> {
|
||||
let mut buffer: [u8; 65535] = [0; 65535];
|
||||
|
||||
let sel_str = match selection {
|
||||
Selection::FirstId => "1",
|
||||
Selection::SecondId => "2",
|
||||
Selection::All => "1:*",
|
||||
};
|
||||
|
||||
let kind_str = match kind {
|
||||
FetchKind::Rfc822 => "RFC822",
|
||||
FetchKind::Rfc822Size => "RFC822.SIZE",
|
||||
};
|
||||
|
||||
let mod_str = match modifier {
|
||||
FetchMod::None => "".into(),
|
||||
FetchMod::ChangedSince(val) => format!(" (CHANGEDSINCE {})", val),
|
||||
};
|
||||
|
||||
imap.write(format!("40 fetch {} {}{}\r\n", sel_str, kind_str, mod_str).as_bytes())?;
|
||||
|
||||
let read = read_lines(imap, &mut buffer, Some(&b"40 OK FETCH"[..]))?;
|
||||
let srv_msg = std::str::from_utf8(read)?;
|
||||
|
||||
Ok(srv_msg.to_string())
|
||||
}
|
||||
|
||||
pub fn copy(imap: &mut TcpStream, selection: Selection, to: Mailbox) -> Result<String> {
|
||||
let mut buffer: [u8; 65535] = [0; 65535];
|
||||
assert!(matches!(selection, Selection::FirstId));
|
||||
assert!(matches!(to, Mailbox::Archive));
|
||||
|
||||
imap.write(&b"45 copy 1 ArchiveCustom\r\n"[..])?;
|
||||
let read = read_lines(imap, &mut buffer, None)?;
|
||||
assert_eq!(&read[..5], &b"45 OK"[..]);
|
||||
let srv_msg = std::str::from_utf8(read)?;
|
||||
|
||||
Ok(srv_msg.to_string())
|
||||
}
|
||||
|
||||
pub fn append(imap: &mut TcpStream, content: Email) -> Result<String> {
|
||||
let mut buffer: [u8; 6000] = [0; 6000];
|
||||
|
||||
let ref_mail = match content {
|
||||
Email::Multipart => EMAIL1,
|
||||
Email::Basic => EMAIL2,
|
||||
};
|
||||
|
||||
let append_cmd = format!("47 append inbox (\\Seen) {{{}}}\r\n", ref_mail.len());
|
||||
println!("append cmd: {}", append_cmd);
|
||||
imap.write(append_cmd.as_bytes())?;
|
||||
|
||||
// wait for continuation
|
||||
let read = read_lines(imap, &mut buffer, None)?;
|
||||
assert_eq!(read[0], b'+');
|
||||
|
||||
// write our stuff
|
||||
imap.write(ref_mail)?;
|
||||
imap.write(&b"\r\n"[..])?;
|
||||
let read = read_lines(imap, &mut buffer, Some(&b"47 OK"[..]))?;
|
||||
let srv_msg = std::str::from_utf8(read)?;
|
||||
|
||||
Ok(srv_msg.to_string())
|
||||
}
|
||||
|
||||
pub fn search(imap: &mut TcpStream, sk: SearchKind) -> Result<String> {
|
||||
let sk_str = match sk {
|
||||
SearchKind::Text(x) => format!("TEXT \"{}\"", x),
|
||||
SearchKind::ModSeq(x) => format!("MODSEQ {}", x),
|
||||
};
|
||||
imap.write(format!("55 SEARCH {}\r\n", sk_str).as_bytes())?;
|
||||
let mut buffer: [u8; 1500] = [0; 1500];
|
||||
let read = read_lines(imap, &mut buffer, Some(&b"55 OK"[..]))?;
|
||||
let srv_msg = std::str::from_utf8(read)?;
|
||||
Ok(srv_msg.to_string())
|
||||
}
|
||||
|
||||
pub fn store(
|
||||
imap: &mut TcpStream,
|
||||
sel: Selection,
|
||||
flag: Flag,
|
||||
action: StoreAction,
|
||||
modifier: StoreMod,
|
||||
) -> Result<String> {
|
||||
let mut buffer: [u8; 6000] = [0; 6000];
|
||||
|
||||
let seq = match sel {
|
||||
Selection::FirstId => "1",
|
||||
Selection::SecondId => "2",
|
||||
Selection::All => "1:*",
|
||||
};
|
||||
|
||||
let modif = match modifier {
|
||||
StoreMod::None => "".into(),
|
||||
StoreMod::UnchangedSince(val) => format!(" (UNCHANGEDSINCE {})", val),
|
||||
};
|
||||
|
||||
let flags_str = match flag {
|
||||
Flag::Deleted => "(\\Deleted)",
|
||||
Flag::Important => "(\\Important)",
|
||||
};
|
||||
|
||||
let action_str = match action {
|
||||
StoreAction::AddFlags => "+FLAGS",
|
||||
StoreAction::DelFlags => "-FLAGS",
|
||||
StoreAction::SetFlags => "FLAGS",
|
||||
StoreAction::AddFlagsSilent => "+FLAGS.SILENT",
|
||||
StoreAction::DelFlagsSilent => "-FLAGS.SILENT",
|
||||
StoreAction::SetFlagsSilent => "FLAGS.SILENT",
|
||||
};
|
||||
|
||||
imap.write(format!("57 STORE {}{} {} {}\r\n", seq, modif, action_str, flags_str).as_bytes())?;
|
||||
let read = read_lines(imap, &mut buffer, Some(&b"57 OK"[..]))?;
|
||||
let srv_msg = std::str::from_utf8(read)?;
|
||||
Ok(srv_msg.to_string())
|
||||
}
|
||||
|
||||
pub fn expunge(imap: &mut TcpStream) -> Result<()> {
|
||||
imap.write(&b"60 expunge\r\n"[..])?;
|
||||
let mut buffer: [u8; 1500] = [0; 1500];
|
||||
let _read = read_lines(imap, &mut buffer, Some(&b"60 OK EXPUNGE"[..]))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn uid_expunge(imap: &mut TcpStream, sel: Selection) -> Result<String> {
|
||||
use Selection::*;
|
||||
let mut buffer: [u8; 6000] = [0; 6000];
|
||||
let selstr = match sel {
|
||||
FirstId => "1",
|
||||
SecondId => "2",
|
||||
All => "1:*",
|
||||
};
|
||||
imap.write(format!("61 UID EXPUNGE {}\r\n", selstr).as_bytes())?;
|
||||
let read = read_lines(imap, &mut buffer, Some(&b"61 OK"[..]))?;
|
||||
let srv_msg = std::str::from_utf8(read)?;
|
||||
Ok(srv_msg.to_string())
|
||||
}
|
||||
|
||||
pub fn rename_mailbox(imap: &mut TcpStream, from: Mailbox, to: Mailbox) -> Result<()> {
|
||||
assert!(matches!(from, Mailbox::Archive));
|
||||
assert!(matches!(to, Mailbox::Drafts));
|
||||
|
||||
imap.write(&b"70 rename ArchiveCustom DraftsCustom\r\n"[..])?;
|
||||
let mut buffer: [u8; 1500] = [0; 1500];
|
||||
let read = read_lines(imap, &mut buffer, None)?;
|
||||
assert_eq!(&read[..5], &b"70 OK"[..]);
|
||||
|
||||
imap.write(&b"71 list \"\" *\r\n"[..])?;
|
||||
let read = read_lines(imap, &mut buffer, Some(&b"71 OK LIST"[..]))?;
|
||||
let srv_msg = std::str::from_utf8(read)?;
|
||||
assert!(!srv_msg.contains(" ArchiveCustom\r\n"));
|
||||
assert!(srv_msg.contains(" INBOX\r\n"));
|
||||
assert!(srv_msg.contains(" DraftsCustom\r\n"));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn delete_mailbox(imap: &mut TcpStream, mbx: Mailbox) -> Result<()> {
|
||||
let mbx_str = match mbx {
|
||||
Mailbox::Inbox => "INBOX",
|
||||
Mailbox::Archive => "ArchiveCustom",
|
||||
Mailbox::Drafts => "DraftsCustom",
|
||||
};
|
||||
let cmd = format!("80 delete {}\r\n", mbx_str);
|
||||
|
||||
imap.write(cmd.as_bytes())?;
|
||||
let mut buffer: [u8; 1500] = [0; 1500];
|
||||
let read = read_lines(imap, &mut buffer, None)?;
|
||||
assert_eq!(&read[..5], &b"80 OK"[..]);
|
||||
|
||||
imap.write(&b"81 list \"\" *\r\n"[..])?;
|
||||
let read = read_lines(imap, &mut buffer, Some(&b"81 OK"[..]))?;
|
||||
let srv_msg = std::str::from_utf8(read)?;
|
||||
assert!(srv_msg.contains(" INBOX\r\n"));
|
||||
assert!(!srv_msg.contains(format!(" {}\r\n", mbx_str).as_str()));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn close(imap: &mut TcpStream) -> Result<()> {
|
||||
imap.write(&b"60 close\r\n"[..])?;
|
||||
let mut buffer: [u8; 1500] = [0; 1500];
|
||||
let _read = read_lines(imap, &mut buffer, Some(&b"60 OK"[..]))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn r#move(imap: &mut TcpStream, selection: Selection, to: Mailbox) -> Result<String> {
|
||||
let mut buffer: [u8; 1500] = [0; 1500];
|
||||
assert!(matches!(to, Mailbox::Archive));
|
||||
assert!(matches!(selection, Selection::FirstId));
|
||||
|
||||
imap.write(&b"35 move 1 ArchiveCustom\r\n"[..])?;
|
||||
let read = read_lines(imap, &mut buffer, Some(&b"35 OK"[..]))?;
|
||||
let srv_msg = std::str::from_utf8(read)?;
|
||||
assert!(srv_msg.contains("* 1 EXPUNGE"));
|
||||
|
||||
Ok(srv_msg.to_string())
|
||||
}
|
||||
|
||||
pub fn enable(imap: &mut TcpStream, ask: Enable, done: Option<Enable>) -> Result<()> {
|
||||
let mut buffer: [u8; 6000] = [0; 6000];
|
||||
assert!(matches!(ask, Enable::Utf8Accept));
|
||||
|
||||
imap.write(&b"36 enable UTF8=ACCEPT\r\n"[..])?;
|
||||
let read = read_lines(imap, &mut buffer, Some(&b"36 OK"[..]))?;
|
||||
let srv_msg = std::str::from_utf8(read)?;
|
||||
match done {
|
||||
None => assert_eq!(srv_msg.lines().count(), 1),
|
||||
Some(Enable::Utf8Accept) => {
|
||||
assert_eq!(srv_msg.lines().count(), 2);
|
||||
assert!(srv_msg.contains("* ENABLED UTF8=ACCEPT"));
|
||||
}
|
||||
_ => unimplemented!(),
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn start_idle(imap: &mut TcpStream) -> Result<()> {
|
||||
let mut buffer: [u8; 1500] = [0; 1500];
|
||||
imap.write(&b"98 IDLE\r\n"[..])?;
|
||||
let read = read_lines(imap, &mut buffer, None)?;
|
||||
assert_eq!(read[0], b'+');
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn stop_idle(imap: &mut TcpStream) -> Result<String> {
|
||||
let mut buffer: [u8; 16536] = [0; 16536];
|
||||
imap.write(&b"DONE\r\n"[..])?;
|
||||
let read = read_lines(imap, &mut buffer, Some(&b"98 OK"[..]))?;
|
||||
let srv_msg = std::str::from_utf8(read)?;
|
||||
Ok(srv_msg.to_string())
|
||||
}
|
||||
|
||||
pub fn logout(imap: &mut TcpStream) -> Result<()> {
|
||||
imap.write(&b"99 logout\r\n"[..])?;
|
||||
let mut buffer: [u8; 1500] = [0; 1500];
|
||||
let read = read_lines(imap, &mut buffer, None)?;
|
||||
assert_eq!(&read[..5], &b"* BYE"[..]);
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -1,99 +0,0 @@
|
|||
#![allow(dead_code)]
|
||||
pub mod constants;
|
||||
pub mod fragments;
|
||||
|
||||
use anyhow::{bail, Context, Result};
|
||||
use std::io::Read;
|
||||
use std::net::{Shutdown, TcpStream};
|
||||
use std::process::Command;
|
||||
use std::thread;
|
||||
|
||||
use constants::SMALL_DELAY;
|
||||
|
||||
pub fn aerogramme_provider_daemon_dev(
|
||||
mut fx: impl FnMut(&mut TcpStream, &mut TcpStream) -> Result<()>,
|
||||
) -> Result<()> {
|
||||
// Check port is not used (= free) before starting the test
|
||||
let mut max_retry = 20;
|
||||
loop {
|
||||
max_retry -= 1;
|
||||
match (TcpStream::connect("[::1]:1143"), max_retry) {
|
||||
(Ok(_), 0) => bail!("something is listening on [::1]:1143 and prevent the test from starting"),
|
||||
(Ok(_), _) => println!("something is listening on [::1]:1143, maybe a previous daemon quitting, retrying soon..."),
|
||||
(Err(_), _) => {
|
||||
println!("test ready to start, [::1]:1143 is free!");
|
||||
break
|
||||
}
|
||||
}
|
||||
thread::sleep(SMALL_DELAY);
|
||||
}
|
||||
|
||||
// Start daemon
|
||||
let mut daemon = Command::new(env!("CARGO_BIN_EXE_aerogramme"))
|
||||
.arg("--dev")
|
||||
.arg("provider")
|
||||
.arg("daemon")
|
||||
.spawn()?;
|
||||
|
||||
// Check that our daemon is correctly listening on the free port
|
||||
let mut max_retry = 20;
|
||||
let mut imap_socket = loop {
|
||||
max_retry -= 1;
|
||||
match (TcpStream::connect("[::1]:1143"), max_retry) {
|
||||
(Err(e), 0) => bail!("no more retry, last error is: {}", e),
|
||||
(Err(e), _) => {
|
||||
println!("unable to connect: {} ; will retry soon...", e);
|
||||
}
|
||||
(Ok(v), _) => break v,
|
||||
}
|
||||
thread::sleep(SMALL_DELAY);
|
||||
};
|
||||
|
||||
// Assuming now it's safe to open a LMTP socket
|
||||
let mut lmtp_socket =
|
||||
TcpStream::connect("[::1]:1025").context("lmtp socket must be connected")?;
|
||||
|
||||
println!("-- ready to test imap features --");
|
||||
let result = fx(&mut imap_socket, &mut lmtp_socket);
|
||||
println!("-- test teardown --");
|
||||
|
||||
imap_socket
|
||||
.shutdown(Shutdown::Both)
|
||||
.context("closing imap socket at the end of the test")?;
|
||||
lmtp_socket
|
||||
.shutdown(Shutdown::Both)
|
||||
.context("closing lmtp socket at the end of the test")?;
|
||||
daemon.kill().context("daemon should be killed")?;
|
||||
|
||||
result.context("all tests passed")
|
||||
}
|
||||
|
||||
pub fn read_lines<'a, F: Read>(
|
||||
reader: &mut F,
|
||||
buffer: &'a mut [u8],
|
||||
stop_marker: Option<&[u8]>,
|
||||
) -> Result<&'a [u8]> {
|
||||
let mut nbytes = 0;
|
||||
loop {
|
||||
nbytes += reader.read(&mut buffer[nbytes..])?;
|
||||
//println!("partial read: {}", std::str::from_utf8(&buffer[..nbytes])?);
|
||||
let pre_condition = match stop_marker {
|
||||
None => true,
|
||||
Some(mark) => buffer[..nbytes].windows(mark.len()).any(|w| w == mark),
|
||||
};
|
||||
if pre_condition && nbytes >= 2 && &buffer[nbytes - 2..nbytes] == &b"\r\n"[..] {
|
||||
break;
|
||||
}
|
||||
}
|
||||
println!("read: {}", std::str::from_utf8(&buffer[..nbytes])?);
|
||||
Ok(&buffer[..nbytes])
|
||||
}
|
||||
|
||||
pub fn read_first_u32(inp: &str) -> Result<u32> {
|
||||
Ok(inp
|
||||
.chars()
|
||||
.skip_while(|c| !c.is_digit(10))
|
||||
.take_while(|c| c.is_digit(10))
|
||||
.collect::<String>()
|
||||
.parse::<u32>()?)
|
||||
}
|
||||
1
tests/emails/.gitattributes
vendored
1
tests/emails/.gitattributes
vendored
|
|
@ -1 +0,0 @@
|
|||
*.zstd filter=lfs diff=lfs merge=lfs -text
|
||||
1
tests/emails/.gitignore
vendored
1
tests/emails/.gitignore
vendored
|
|
@ -1 +0,0 @@
|
|||
*.mbox
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
Emails from the rfc/ legacy/ malformed/ and thirdparty/ folders were imported from this repository:
|
||||
https://github.com/stalwartlabs/mail-parser
|
||||
Specific COPYING files are available in each folder to trace their authors.
|
||||
If nothing is stated in these folders, the following copyright applies: Copyright (C) 2020-2022, Stalwart Labs Ltd.
|
||||
And the distribution license is either BSD-2, MIT or Apache 2.
|
||||
|
||||
BIN
tests/emails/aero100.mbox.zstd
(Stored with Git LFS)
BIN
tests/emails/aero100.mbox.zstd
(Stored with Git LFS)
Binary file not shown.
|
|
@ -1 +0,0 @@
|
|||
(BODY ("TEXT" "PLAIN" ("CHARSET" "us-ascii") NIL NIL "7BIT" 49 1))
|
||||
|
|
@ -1 +0,0 @@
|
|||
(BODYSTRUCTURE ("TEXT" "PLAIN" ("CHARSET" "us-ascii") NIL NIL "7BIT" 49 1 NIL NIL NIL NIL))
|
||||
|
|
@ -1 +0,0 @@
|
|||
(BODY ("text" "plain" ("charset" "us-ascii") NIL NIL "7bit" 49 1))
|
||||
|
|
@ -1 +0,0 @@
|
|||
(BODYSTRUCTURE ("text" "plain" ("charset" "us-ascii") NIL NIL "7bit" 49 1 NIL NIL NIL NIL))
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
From: Garage team <garagehq@deuxfleurs.fr>
|
||||
Subject: Welcome to Aerogramme!!
|
||||
|
||||
This is just a test email, feel free to ignore.
|
||||
|
|
@ -1 +0,0 @@
|
|||
(BODY ("text" "plain" () NIL NIL NIL 49 1))
|
||||
|
|
@ -1 +0,0 @@
|
|||
(BODYSTRUCTURE ("text" "plain" () NIL NIL NIL 49 1 NIL NIL NIL NIL))
|
||||
|
|
@ -1 +0,0 @@
|
|||
(BODY ("text" "plain" ("charset" "us-ascii") NIL NIL "7bit" 49 1))
|
||||
|
|
@ -1 +0,0 @@
|
|||
(BODYSTRUCTURE ("text" "plain" ("charset" "us-ascii") NIL NIL "7bit" 49 1 "ffd9b6292b7ea945513c94e06a2ce185" NIL NIL NIL))
|
||||
|
|
@ -1 +0,0 @@
|
|||
(BODY (("TEXT" "PLAIN" ("CHARSET" "us-ascii") NIL NIL "7BIT" 9 1)("TEXT" "HTML" ("CHARSET" "utf-8") NIL NIL "8BIT" 14 0) "ALTERNATIVE"))
|
||||
|
|
@ -1 +0,0 @@
|
|||
(BODYSTRUCTURE (("TEXT" "PLAIN" ("CHARSET" "us-ascii") NIL NIL "7BIT" 9 1 NIL NIL NIL NIL)("TEXT" "HTML" ("CHARSET" "utf-8") NIL NIL "8BIT" 14 0 NIL NIL NIL NIL) "ALTERNATIVE" ("BOUNDARY" "festivus") NIL NIL NIL))
|
||||
|
|
@ -1 +0,0 @@
|
|||
(BODY (("text" "plain" ("charset" "us-ascii") NIL NIL "7bit" 9 1)("text" "html" ("charset" "utf-8") NIL NIL "8bit" 14 0) "alternative"))
|
||||
|
|
@ -1 +0,0 @@
|
|||
(BODYSTRUCTURE (("text" "plain" ("charset" "us-ascii") NIL NIL "7bit" 9 1 NIL NIL NIL NIL)("text" "html" ("charset" "utf-8") NIL NIL "8bit" 14 0 NIL NIL NIL NIL) "alternative" ("boundary" "festivus") NIL NIL NIL))
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue