From 6d283b8e114dacd47d8d6be8ee7bfdbc0c3caed7 Mon Sep 17 00:00:00 2001 From: Rae McKelvey <633012+okdistribute@users.noreply.github.com> Date: Mon, 16 Mar 2026 11:46:30 -0700 Subject: [PATCH 1/7] noq instead of quinn --- connecting/creating-endpoint.mdx | 4 ++-- connecting/dns-discovery.mdx | 4 ++-- connecting/gossip.mdx | 4 ++-- examples/chat.mdx | 12 ++++++------ protocols/blobs.mdx | 4 ++-- protocols/rpc.mdx | 18 +++++++++--------- protocols/writing-a-protocol.mdx | 4 ++-- quickstart.mdx | 18 +++++++++--------- 8 files changed, 34 insertions(+), 34 deletions(-) diff --git a/connecting/creating-endpoint.mdx b/connecting/creating-endpoint.mdx index 8a3f65c..09c134c 100644 --- a/connecting/creating-endpoint.mdx +++ b/connecting/creating-endpoint.mdx @@ -15,11 +15,11 @@ This method initializes a new endpoint and binds it to a local address, allowing to listen for incoming connections. ```rust -use iroh::Endpoint; +use iroh::{Endpoint, presets}; #[tokio::main] async fn main() { - let endpoint = Endpoint::bind().await?; + let endpoint = Endpoint::bind(presets::N0).await?; // ... } ``` diff --git a/connecting/dns-discovery.mdx b/connecting/dns-discovery.mdx index 8988d4e..6e02f7c 100644 --- a/connecting/dns-discovery.mdx +++ b/connecting/dns-discovery.mdx @@ -25,12 +25,12 @@ options](https://cal.com/team/number-0/n0-protocol-services?overlayCalendar=true ```rust -use iroh::Endpoint; +use iroh::{Endpoint, presets}; use iroh_tickets::endpoint::EndpointTicket; #[tokio::main] async fn main() -> anyhow::Result<()> { - let endpoint = Endpoint::bind().await?; + let endpoint = Endpoint::bind(presets::N0).await?; println!("endpoint id: {:?}", endpoint.id()); diff --git a/connecting/gossip.mdx b/connecting/gossip.mdx index dc0be9e..c85f6ba 100644 --- a/connecting/gossip.mdx +++ b/connecting/gossip.mdx @@ -47,7 +47,7 @@ There are different ways to structure your application around topics, depending ### Example ```rust -use iroh::{protocol::Router, Endpoint, EndpointId}; +use iroh::{protocol::Router, Endpoint, EndpointId, presets}; use iroh_gossip::{api::Event, Gossip, TopicId}; use n0_error::{Result, StdResultExt}; use n0_future::StreamExt; @@ -56,7 +56,7 @@ use n0_future::StreamExt; async fn main() -> Result<()> { // create an iroh endpoint that includes the standard discovery mechanisms // we've built at number0 - let endpoint = Endpoint::bind().await?; + let endpoint = Endpoint::bind(presets::N0).await?; // build gossip protocol let gossip = Gossip::builder().spawn(endpoint.clone()); diff --git a/examples/chat.mdx b/examples/chat.mdx index 3d413b0..9d9e1a7 100644 --- a/examples/chat.mdx +++ b/examples/chat.mdx @@ -134,12 +134,12 @@ Topics are the fundamental unit of communication in the gossip protocol. Here's ```rust use anyhow::Result; use iroh::protocol::Router; -use iroh::Endpoint; +use iroh::{Endpoint, presets}; use iroh_gossip::{net::Gossip, proto::TopicId}; #[tokio::main] async fn main() -> Result<()> { - let endpoint = Endpoint::bind().await?; + let endpoint = Endpoint::bind(presets::N0).await?; println!("> our endpoint id: {}", endpoint.id()); let gossip = Gossip::builder().spawn(endpoint.clone()); @@ -324,7 +324,7 @@ use std::collections::HashMap; use anyhow::Result; use futures_lite::StreamExt; use iroh::protocol::Router; -use iroh::{Endpoint, EndpointId}; +use iroh::{Endpoint, EndpointId, presets}; use iroh_gossip::{ api::{GossipReceiver, Event}, net::Gossip, @@ -334,7 +334,7 @@ use serde::{Deserialize, Serialize}; #[tokio::main] async fn main() -> Result<()> { - let endpoint = Endpoint::bind().await?; + let endpoint = Endpoint::bind(presets::N0).await?; println!("> our endpoint id: {}", endpoint.id()); let gossip = Gossip::builder().spawn(endpoint.clone()); @@ -535,7 +535,7 @@ use std::{collections::HashMap, fmt, str::FromStr}; use anyhow::Result; use clap::Parser; use futures_lite::StreamExt; -use iroh::{protocol::Router, Endpoint, EndpointAddr, EndpointId}; +use iroh::{protocol::Router, Endpoint, EndpointAddr, EndpointId, presets}; use iroh_gossip::{ api::{GossipReceiver, Event}, net::Gossip, @@ -591,7 +591,7 @@ async fn main() -> Result<()> { } }; - let endpoint = Endpoint::bind().await?; + let endpoint = Endpoint::bind(presets::N0).await?; println!("> our endpoint id: {}", endpoint.id()); let gossip = Gossip::builder().spawn(endpoint.clone()); diff --git a/protocols/blobs.mdx b/protocols/blobs.mdx index c00836b..53a499a 100644 --- a/protocols/blobs.mdx +++ b/protocols/blobs.mdx @@ -104,7 +104,7 @@ This is what manages the possibly changing network underneath, maintains a conne async fn main() -> anyhow::Result<()> { // Create an endpoint, it allows creating and accepting // connections in the iroh p2p world - let endpoint = Endpoint::bind().await?; + let endpoint = Endpoint::bind(presets::N0).await?; // ... @@ -124,7 +124,7 @@ It loads files from your file system and provides a protocol for seekable, resum async fn main() -> anyhow::Result<()> { // Create an endpoint, it allows creating and accepting // connections in the iroh p2p world - let endpoint = Endpoint::bind().await?; + let endpoint = Endpoint::bind(presets::N0).await?; // We initialize an in-memory backing store for iroh-blobs let store = MemStore::new(); diff --git a/protocols/rpc.mdx b/protocols/rpc.mdx index ce9fb06..4fba995 100644 --- a/protocols/rpc.mdx +++ b/protocols/rpc.mdx @@ -169,7 +169,7 @@ But we said that we wanted to be able to seamlessly switch between remote or loc ```rust enum Client { Local(mpsc::Sender), - Remote(quinn::Connection), + Remote(noq::Connection), } impl Client { @@ -206,7 +206,7 @@ But what about all this boilerplate? **The `irpc` crate is meant solely to reduce the tedious boilerplate involved in writing the above manually.** -It does *not* abstract over the connection type - it only supports [iroh-quinn] send and receive streams out of the box, so the only two possible connection types are `iroh` p2p QUIC connections and normal QUIC connections. It also does not abstract over the local channel type - a local channel is always a `tokio::sync::mpsc` channel. Serialization is always using postcard and length prefixes are always postcard varints. +It does *not* abstract over the connection type - it only supports [noq] QUIC send and receive streams out of the box, so the only two possible connection types are `iroh` p2p QUIC connections and normal QUIC connections. It also does not abstract over the local channel type - a local channel is always a `tokio::sync::mpsc` channel. Serialization is always using postcard and length prefixes are always postcard varints. So let's see what our kv service looks like using `irpc`: @@ -286,8 +286,8 @@ converting the result into a futures `Stream` or the updates into a futures services that can be used in-process or across processes, not to provide an opinionated high level API. -For stream based rpc calls, there is an issue you should be aware of. The quinn -`SendStream` will send a finish message when dropped. So if you have a finite +For stream based rpc calls, there is an issue you should be aware of. The noq +QUIC `SendStream` will send a finish message when dropped. So if you have a finite stream, you might want to have an explicit end marker that you send before dropping the sender to allow the remote side to distinguish between successful termination and abnormal termination. E.g. the `SetFromStream` request from @@ -330,9 +330,9 @@ If you are reading from a remote source, and there is a problem with the connect But what about writing? E.g. you got a task that performs an expensive computation and writes updates to the remote in regular intervals. You will only detect that the remote side is gone once you write, so if you write infrequently you will perform an expensive computation despite the remote side no longer being available or interested. -To solve this, an irpc Sender has a [closed](https://docs.rs/irpc/0.5.0/irpc/channel/mpsc/enum.Sender.html#method.closed) function that you can use to detect the remote closing without having to send a message. This wraps [tokio::sync::mpsc::Sender::closed](https://docs.rs/tokio/latest/tokio/sync/mpsc/struct.Sender.html#method.closed) for local streams and [quinn::SendStream::stopped](https://docs.rs/iroh-quinn/latest/iroh_quinn/struct.SendStream.html#method.stopped) for remote streams. +To solve this, an irpc Sender has a [closed](https://docs.rs/irpc/0.5.0/irpc/channel/mpsc/enum.Sender.html#method.closed) function that you can use to detect the remote closing without having to send a message. This wraps [tokio::sync::mpsc::Sender::closed](https://docs.rs/tokio/latest/tokio/sync/mpsc/struct.Sender.html#method.closed) for local streams and [noq::SendStream::stopped](https://github.com/n0-computer/noq) for remote QUIC streams. -## Alternatives to iroh-quinn +## Alternatives to noq If you integrate iroh protocols into an existing application, it could be that you already have a rpc system that you are happy with, like [grpc](https://grpc.io/) or [json-rpc](https://www.jsonrpc.org/). @@ -361,9 +361,9 @@ and maintained. ## References - [postcard](https://docs.rs/postcard/latest/postcard/) -- [iroh-quinn](https://docs.rs/iroh-quinn/latest/iroh_quinn/) -- [RecvStream](https://docs.rs/iroh-quinn/latest/iroh_quinn/struct.RecvStream.html) -- [SendStream](https://docs.rs/iroh-quinn/latest/iroh_quinn/struct.SendStream.html) +- [noq](https://github.com/n0-computer/noq) +- [RecvStream](https://github.com/n0-computer/noq) +- [SendStream](https://github.com/n0-computer/noq) - [Stream](https://docs.rs/futures/latest/futures/prelude/trait.Stream.html) - [Sink](https://docs.rs/futures/latest/futures/sink/trait.Sink.html) - [snafu](https://docs.rs/snafu/latest/snafu/) diff --git a/protocols/writing-a-protocol.mdx b/protocols/writing-a-protocol.mdx index b0145e8..2b3cab8 100644 --- a/protocols/writing-a-protocol.mdx +++ b/protocols/writing-a-protocol.mdx @@ -87,7 +87,7 @@ Now, we can modify our router so it handles incoming connections with our newly ```rs async fn start_accept_side() -> anyhow::Result { - let endpoint = iroh::Endpoint::bind().await?; + let endpoint = iroh::Endpoint::bind(iroh::presets::N0).await?; let router = iroh::protocol::Router::builder(endpoint) .accept(ALPN, Echo) // This makes the router handle incoming connections with our ALPN via Echo::accept! @@ -164,7 +164,7 @@ This follows the [request-response pattern](/protocols/using-quic#request-and-re ```rs async fn connect_side(addr: EndpointAddr) -> Result<()> { - let endpoint = Endpoint::bind().await?; + let endpoint = Endpoint::bind(presets::N0).await?; // Open a connection to the accepting endpoint let conn = endpoint.connect(addr, ALPN).await?; diff --git a/quickstart.mdx b/quickstart.mdx index e78583c..e65c805 100644 --- a/quickstart.mdx +++ b/quickstart.mdx @@ -41,7 +41,7 @@ connection to the closest relay, and finds ways to address devices by async fn main() -> anyhow::Result<()> { // Create an endpoint, it allows creating and accepting // connections in the iroh p2p world - let endpoint = Endpoint::bind().await?; + let endpoint = Endpoint::bind(presets::N0).await?; // ... @@ -81,14 +81,14 @@ round-trip latency, or whatever else you want to build on top of it, without bui ```rust use anyhow::Result; -use iroh::{protocol::Router, Endpoint, Watcher}; +use iroh::{protocol::Router, Endpoint, Watcher, presets}; use iroh_ping::Ping; #[tokio::main] async fn main() -> anyhow::Result<()> { // Create an endpoint, it allows creating and accepting // connections in the iroh p2p world - let endpoint = Endpoint::bind().await?; + let endpoint = Endpoint::bind(presets::N0).await?; // bring the endpoint online before accepting connections endpoint.online().await; @@ -116,14 +116,14 @@ arguments and match on them: ```rust use anyhow::Result; -use iroh::{protocol::Router, Endpoint, Watcher}; +use iroh::{protocol::Router, Endpoint, Watcher, presets}; use iroh_ping::Ping; #[tokio::main] async fn main() -> Result<()> { // Create an endpoint, it allows creating and accepting // connections in the iroh p2p world - let endpoint = Endpoint::bind().await?; + let endpoint = Endpoint::bind(presets::N0).await?; // Then we initialize a struct that can accept ping requests over iroh connections let ping = Ping::new(); @@ -137,7 +137,7 @@ async fn main() -> Result<()> { let addr = recv_router.endpoint().addr(); // create a send side & send a ping - let send_ep = Endpoint::bind().await?; + let send_ep = Endpoint::bind(presets::N0).await?; let send_pinger = Ping::new(); let rtt = send_pinger.ping(&send_ep, addr).await?; @@ -207,7 +207,7 @@ a router that accepts incoming ping requests indefinitely: ```rust // filepath: src/main.rs use anyhow::{anyhow, Result}; -use iroh::{Endpoint, protocol::Router}; +use iroh::{Endpoint, protocol::Router, presets}; use iroh_ping::Ping; use iroh_tickets::{Ticket, endpoint::EndpointTicket}; use std::env; @@ -215,7 +215,7 @@ use std::env; async fn run_receiver() -> Result<()> { // Create an endpoint, it allows creating and accepting // connections in the iroh p2p world - let endpoint = Endpoint::bind().await?; + let endpoint = Endpoint::bind(presets::N0).await?; // Wait for the endpoint to be accessible by others on the internet endpoint.online().await; @@ -245,7 +245,7 @@ The sender parses the ticket, creates its own endpoint, and pings the receiver's ```rust // filepath: src/main.rs async fn run_sender(ticket: EndpointTicket) -> Result<()> { - let send_ep = Endpoint::bind().await?; + let send_ep = Endpoint::bind(presets::N0).await?; let send_pinger = Ping::new(); let rtt = send_pinger.ping(&send_ep, ticket.endpoint_addr().clone()).await?; println!("ping took: {:?} to complete", rtt); From e9a1b0992c415fde4fe814310aaa265623fe1895 Mon Sep 17 00:00:00 2001 From: Rae McKelvey <633012+okdistribute@users.noreply.github.com> Date: Tue, 17 Mar 2026 10:15:05 -0700 Subject: [PATCH 2/7] add transports --- connecting/local-discovery.mdx | 45 ----------------------- docs.json | 11 +++++- protocols/using-quic.md | 2 +- transports/bluetooth.mdx | 36 ++++++++++++++++++ transports/mdns.mdx | 40 ++++++++++++++++++++ transports/nym.mdx | 59 ++++++++++++++++++++++++++++++ transports/quic.mdx | 42 +++++++++++++++++++++ transports/tor.mdx | 67 ++++++++++++++++++++++++++++++++++ 8 files changed, 255 insertions(+), 47 deletions(-) delete mode 100644 connecting/local-discovery.mdx create mode 100644 transports/bluetooth.mdx create mode 100644 transports/mdns.mdx create mode 100644 transports/nym.mdx create mode 100644 transports/quic.mdx create mode 100644 transports/tor.mdx diff --git a/connecting/local-discovery.mdx b/connecting/local-discovery.mdx deleted file mode 100644 index 0f86d2e..0000000 --- a/connecting/local-discovery.mdx +++ /dev/null @@ -1,45 +0,0 @@ ---- -title: "mDNS and Bluetooth" ---- - -Local discovery adds the ability to use physical radios to discover other iroh -endpoints. This is useful for local networks where the internet may not be available or reliable. - -Local connections can be faster and more reliable than internet-based connections, especially in -environments with poor connectivity. They also enhance privacy by keeping communications within a local area. - -## mDNS - -Local Discovery is _not_ enabled by default, and must be enabled by the user. -You'll need to add the `discovery-local-network` feature flag to your -`Cargo.toml` to use it. - -```toml -[dependencies] -# Make sure to use the most recent version here instead of nn. (at the time of writing: 0.32) -iroh = { version = "0.nn", features = ["address-lookup-mdns"] } -``` - -Then configure your endpoint to use local discovery concurrently with the default DNS discovery: - -```rust -use iroh::Endpoint; - -let mdns = iroh::address_lookup::mdns::MdnsAddressLookup::builder(); -let ep = Endpoint::builder() - .address_lookup(mdns) - .bind() - .await?; -``` - -The mDNS discovery mechanism will automatically broadcast your endpoint's -presence on the local network, and listen for other endpoints doing the same. When -another endpoint is discovered, the dialing information is exchanged, and a -connection can be established directly over the local network without needing a relay. - -For more information on how mDNS discovery works, see the [mDNS documentation](https://docs.rs/iroh/latest/iroh/address_lookup/mdns/index.html). - -## Bluetooth - -Bluetooth discovery is currently under development and will be available in a -future release of iroh. For more information, please [contact us](https://cal.com/team/number-0/n0-protocol-services). diff --git a/docs.json b/docs.json index 81ec2f1..a0aa73d 100644 --- a/docs.json +++ b/docs.json @@ -40,11 +40,20 @@ "connecting/custom-relays", "connecting/dns-discovery", "connecting/dht-discovery", - "connecting/local-discovery", "connecting/gossip", "connecting/endpoint-hooks" ] }, + { + "group": "Transports", + "pages": [ + "transports/quic", + "transports/mdns", + "transports/tor", + "transports/nym", + "transports/bluetooth" + ] + }, { "group": "Building your App", "pages": [ diff --git a/protocols/using-quic.md b/protocols/using-quic.md index 0d58aba..9a70209 100644 --- a/protocols/using-quic.md +++ b/protocols/using-quic.md @@ -15,7 +15,7 @@ Many developers reach for iroh expecting it to completely abstract away the unde Think of iroh as giving you **reliable, secure tunnels between peers**. This guide shows you how to use QUIC's streaming patterns to build efficient protocols inside those tunnels. Whether you're adapting an existing protocol or designing something new, understanding these patterns will help you make the most of iroh's capabilities. -iroh uses a fork of [Quinn](https://docs.rs/iroh-quinn/latest/iroh_quinn/), a pure-Rust implementation of QUIC maintained by [n0.computer](https://n0.computer). Quinn is production-ready, actively maintained, and used by projects beyond iroh. If you need lower-level QUIC access or want to understand the implementation details, check out the [Quinn documentation](https://docs.rs/iroh-quinn/latest/iroh_quinn/). +iroh uses [noq](https://github.com/n0-computer/noq), a pure-Rust QUIC implementation maintained by [n0.computer](https://n0.computer). noq is production-ready, actively maintained, and used by projects beyond iroh. If you need lower-level QUIC access or want to understand the implementation details, check out the [noq repository](https://github.com/n0-computer/noq). diff --git a/transports/bluetooth.mdx b/transports/bluetooth.mdx new file mode 100644 index 0000000..e2d7927 --- /dev/null +++ b/transports/bluetooth.mdx @@ -0,0 +1,36 @@ +--- +title: "Bluetooth (BLE)" +--- + + +Bluetooth Low Energy (BLE) transport support is not yet available. We plan to work with the community to implement it. If you're interested in contributing, reach out on [Discord](https://www.iroh.computer/discord). + + +A BLE transport would allow iroh endpoints to connect directly over Bluetooth Low Energy — useful for local device-to-device scenarios where WiFi or cellular is unavailable, such as offline mesh networking or proximity-based applications. + +## How custom transports work + +When a BLE transport is available, you would add it the same way as any other custom transport: + +```rust +use iroh::presets; + +let endpoint = Endpoint::builder() + .add_custom_transport(ble_transport) + .bind(presets::N0) + .await?; +``` + +You can also combine transports. For example, prefer BLE when nearby peers are available and fall back to QUIC over IP otherwise — iroh will pick the best available path. + +You can only connect to other endpoints that also have the BLE transport enabled. + +## Custom transport API + +The custom transport API lets anyone implement new transports by implementing a set of traits for low-level packet sending and receiving. Each transport defines its own address type and serialization format. + +See [Tor](/transports/tor) and [Nym](/transports/nym) for examples of custom transport implementations today. + + +Custom transport support requires the `unstable-custom-transports` feature flag. The API is unstable and subject to change. See [PR #3845](https://github.com/n0-computer/iroh/pull/3845) for background. + diff --git a/transports/mdns.mdx b/transports/mdns.mdx new file mode 100644 index 0000000..6fc512c --- /dev/null +++ b/transports/mdns.mdx @@ -0,0 +1,40 @@ +--- +title: "mDNS" +--- + +mDNS (Multicast DNS) lets iroh endpoints discover and connect to each other on a local network without needing an internet connection or relay server. + +Local connections can be faster and more reliable than internet-based connections, especially in environments with poor connectivity. They also enhance privacy by keeping communications within a local area. + +Local discovery is _not_ enabled by default and must be opted into. + +## Installation + +Add the `address-lookup-mdns` feature flag: + +```toml +[dependencies] +iroh = { version = "0.nn", features = ["address-lookup-mdns"] } +``` + +## Usage + +Configure your endpoint to use mDNS alongside the default DNS discovery: + +```rust +use iroh::{Endpoint, presets}; + +let mdns = iroh::address_lookup::mdns::MdnsAddressLookup::builder(); +let endpoint = Endpoint::builder() + .address_lookup(mdns) + .bind(presets::N0) + .await?; +``` + +The mDNS mechanism automatically broadcasts your endpoint's presence on the local network and listens for other endpoints doing the same. When another endpoint is discovered, dialing information is exchanged and a connection can be established directly over the local network — no relay needed. + +For more information see the [mDNS API docs](https://docs.rs/iroh/latest/iroh/address_lookup/mdns/index.html). + +## Bluetooth + +Bluetooth Low Energy (BLE) local discovery is currently under development. See the [Bluetooth](/transports/bluetooth) page for more details. diff --git a/transports/nym.mdx b/transports/nym.mdx new file mode 100644 index 0000000..5f205b8 --- /dev/null +++ b/transports/nym.mdx @@ -0,0 +1,59 @@ +--- +title: "Nym" +--- + +The [iroh-nym-transport](https://github.com/n0-computer/iroh-nym-transport) crate routes iroh QUIC packets through the [Nym mixnet](https://nymtech.net/), providing traffic analysis resistance by shuffling and delaying packets across a network of mix nodes. + +This is useful when you need stronger metadata privacy than Tor provides — the mixnet obscures not just your IP address, but also communication patterns and timing. + + +Both iroh's custom transport API and this crate are experimental. Expect breaking changes. + + +## Installation + +```bash +cargo add iroh-nym-transport +``` + +## Usage + +```rust +use std::sync::Arc; +use iroh::{Endpoint, presets}; +use iroh_nym_transport::NymUserTransport; +use nym_sdk::mixnet::MixnetClient; + +let nym_client = MixnetClient::connect_new().await?; +let transport = Arc::new(NymUserTransport::new(nym_client)); + +let endpoint = Endpoint::builder() + .clear_ip_transports() + .add_custom_transport(transport) + .bind(presets::N0) + .await?; +``` + +Calling `clear_ip_transports()` disables QUIC over UDP so all traffic routes exclusively through the mixnet. + +You can only connect to other endpoints that also have the Nym transport enabled. + +## Performance characteristics + +The mixnet intentionally introduces latency and limits throughput as part of its privacy guarantees: + +| Metric | Direct QUIC | Nym mixnet | +|--------|-------------|------------| +| RTT | 50–200 ms | ~1–3 s | +| Throughput | 10+ Mbps | ~15–20 KiB/s | + +Nym is well suited for privacy-sensitive, latency-tolerant applications. It is not suitable for real-time communication or high-throughput file transfer. + +## Feature flag + +Custom transport support must be enabled: + +```toml +[dependencies] +iroh = { version = "*", features = ["unstable-custom-transports"] } +``` diff --git a/transports/quic.mdx b/transports/quic.mdx new file mode 100644 index 0000000..d33a3a0 --- /dev/null +++ b/transports/quic.mdx @@ -0,0 +1,42 @@ +--- +title: "QUIC" +--- + +QUIC is the default transport in iroh. Every endpoint uses QUIC over UDP by default — no configuration required. + +iroh's QUIC implementation is built on [noq](https://github.com/n0-computer/noq), n0's fork of the Quinn QUIC library. Over time the fork diverged significantly as n0 added multipath support, QUIC NAT traversal, and other iroh-specific features. + +All connections are encrypted and authenticated using TLS 1.3. Hole-punching, relay fallback, and multi-path are all handled at the QUIC layer automatically. + +For details on how to use QUIC streams to build protocols on top of an iroh connection, see [Using QUIC](/protocols/using-quic). + +## Custom transports + +QUIC over UDP is the default, but iroh supports plugging in additional custom transports alongside it. See [Tor](/transports/tor), [Nym](/transports/nym), and [Bluetooth](/transports/bluetooth) for examples. + +To add a custom transport while keeping QUIC: + +```rust +use iroh::presets; + +let endpoint = Endpoint::builder() + .add_custom_transport(my_transport) + .bind(presets::N0) + .await?; +``` + +To use a custom transport exclusively and disable QUIC over IP: + +```rust +use iroh::presets; + +let endpoint = Endpoint::builder() + .clear_ip_transports() + .add_custom_transport(my_transport) + .bind(presets::N0) + .await?; +``` + + +Custom transport support requires the `unstable-custom-transports` feature flag. The API is unstable and subject to change. See [PR #3845](https://github.com/n0-computer/iroh/pull/3845) for background. + diff --git a/transports/tor.mdx b/transports/tor.mdx new file mode 100644 index 0000000..11e2716 --- /dev/null +++ b/transports/tor.mdx @@ -0,0 +1,67 @@ +--- +title: "Tor" +--- + +The [iroh-tor-transport](https://github.com/n0-computer/iroh-tor-transport) crate routes iroh connections through [Tor hidden services](https://www.torproject.org/), hiding both peers' IP addresses from each other and from the network. + +This is useful when IP address privacy is a hard requirement for your application. + +For background, see the [blog post](https://www.iroh.computer/blog/tor-custom-transport). + + +Both iroh's custom transport API and this crate are experimental. Expect breaking changes. + + +## Installation + +Add the dependency: + +```bash +cargo add iroh-tor-transport +``` + +You also need a running Tor daemon with the control port enabled: + +```bash +tor --ControlPort 9051 --CookieAuthentication 0 +``` + +## Usage + +```rust +use iroh::{Endpoint, SecretKey, presets}; +use iroh_tor_transport::TorCustomTransport; + +let secret_key = SecretKey::generate(&mut rand::rng()); +let transport = TorCustomTransport::builder() + .build(secret_key.clone()) + .await?; + +let endpoint = Endpoint::builder() + .secret_key(secret_key) + .preset(transport.preset()) + .bind(presets::N0) + .await?; +``` + +You can only connect to other endpoints that also have the Tor transport enabled. + +## How it works + +Connections are routed through Tor onion services. Each endpoint creates a hidden service address, which other endpoints dial over the Tor network. Neither side learns the other's real IP address. + +## Current limitations + +- Requires a separately installed and running Tor daemon +- Cookie authentication must be disabled (`--CookieAuthentication 0`) +- No automatic recovery if the Tor daemon crashes +- Limited to TCP streams over Tor + +## Feature flag + +Custom transport support must be enabled: + +```toml +[dependencies] +iroh = { version = "*", features = ["unstable-custom-transports"] } +``` From 3a72051426c209c7e9760c9a0da2809747b22122 Mon Sep 17 00:00:00 2001 From: Rae McKelvey <633012+okdistribute@users.noreply.github.com> Date: Tue, 17 Mar 2026 10:44:30 -0700 Subject: [PATCH 3/7] add specifics on transports --- docs.json | 7 +- transports/bluetooth.mdx | 27 +-- transports/mdns.mdx | 11 +- transports/quic.mdx | 508 ++++++++++++++++++++++++++++++++++++++- 4 files changed, 510 insertions(+), 43 deletions(-) diff --git a/docs.json b/docs.json index a0aa73d..3497d96 100644 --- a/docs.json +++ b/docs.json @@ -34,7 +34,7 @@ ] }, { - "group": "Forming a Network", + "group": "Creating Connections", "pages": [ "connecting/creating-endpoint", "connecting/custom-relays", @@ -55,7 +55,7 @@ ] }, { - "group": "Building your App", + "group": "Protocols", "pages": [ "protocols/kv-crdts", "protocols/blobs", @@ -63,8 +63,7 @@ "protocols/automerge", "protocols/streaming", "examples/chat", - "protocols/writing-a-protocol", - "protocols/using-quic" + "protocols/writing-a-protocol" ] }, { diff --git a/transports/bluetooth.mdx b/transports/bluetooth.mdx index e2d7927..c22a139 100644 --- a/transports/bluetooth.mdx +++ b/transports/bluetooth.mdx @@ -2,28 +2,9 @@ title: "Bluetooth (BLE)" --- - -Bluetooth Low Energy (BLE) transport support is not yet available. We plan to work with the community to implement it. If you're interested in contributing, reach out on [Discord](https://www.iroh.computer/discord). - - -A BLE transport would allow iroh endpoints to connect directly over Bluetooth Low Energy — useful for local device-to-device scenarios where WiFi or cellular is unavailable, such as offline mesh networking or proximity-based applications. - -## How custom transports work - -When a BLE transport is available, you would add it the same way as any other custom transport: - -```rust -use iroh::presets; - -let endpoint = Endpoint::builder() - .add_custom_transport(ble_transport) - .bind(presets::N0) - .await?; -``` - -You can also combine transports. For example, prefer BLE when nearby peers are available and fall back to QUIC over IP otherwise — iroh will pick the best available path. - -You can only connect to other endpoints that also have the BLE transport enabled. +Bluetooth Low Energy (BLE) transport support is not yet available. We plan to +work with the community to implement it. If you're interested in contributing, +reach out on [Discord](https://www.iroh.computer/discord). ## Custom transport API @@ -31,6 +12,4 @@ The custom transport API lets anyone implement new transports by implementing a See [Tor](/transports/tor) and [Nym](/transports/nym) for examples of custom transport implementations today. - Custom transport support requires the `unstable-custom-transports` feature flag. The API is unstable and subject to change. See [PR #3845](https://github.com/n0-computer/iroh/pull/3845) for background. - diff --git a/transports/mdns.mdx b/transports/mdns.mdx index 6fc512c..22aa823 100644 --- a/transports/mdns.mdx +++ b/transports/mdns.mdx @@ -19,7 +19,7 @@ iroh = { version = "0.nn", features = ["address-lookup-mdns"] } ## Usage -Configure your endpoint to use mDNS alongside the default DNS discovery: +Configure your endpoint to use mDNS alongside the default DNS address lookup: ```rust use iroh::{Endpoint, presets}; @@ -31,10 +31,9 @@ let endpoint = Endpoint::builder() .await?; ``` -The mDNS mechanism automatically broadcasts your endpoint's presence on the local network and listens for other endpoints doing the same. When another endpoint is discovered, dialing information is exchanged and a connection can be established directly over the local network — no relay needed. +The mDNS mechanism automatically broadcasts your endpoint's presence on the +local network and listens for other endpoints doing the same. When another +endpoint is discovered, dialing information is exchanged and a connection can be +established directly over the local network: no relay needed. For more information see the [mDNS API docs](https://docs.rs/iroh/latest/iroh/address_lookup/mdns/index.html). - -## Bluetooth - -Bluetooth Low Energy (BLE) local discovery is currently under development. See the [Bluetooth](/transports/bluetooth) page for more details. diff --git a/transports/quic.mdx b/transports/quic.mdx index d33a3a0..7e28089 100644 --- a/transports/quic.mdx +++ b/transports/quic.mdx @@ -12,18 +12,30 @@ For details on how to use QUIC streams to build protocols on top of an iroh conn ## Custom transports -QUIC over UDP is the default, but iroh supports plugging in additional custom transports alongside it. See [Tor](/transports/tor), [Nym](/transports/nym), and [Bluetooth](/transports/bluetooth) for examples. +QUIC over UDP is the default, but iroh supports plugging in additional custom +transports alongside it. See [Tor](/transports/tor), [Nym](/transports/nym), and +[Bluetooth](/transports/bluetooth) for examples. -To add a custom transport while keeping QUIC: +## Using QUIC -```rust -use iroh::presets; +### Why this matters for iroh -let endpoint = Endpoint::builder() - .add_custom_transport(my_transport) - .bind(presets::N0) - .await?; -``` +iroh is built on top of QUIC, providing connectivity, NAT traversal, and encrypted connections out of the box. While iroh handles the hard parts of networking—holepunching, relay servers, and discovery—**you still need to design how your application exchanges data once connected**. + +Many developers reach for iroh expecting it to completely abstract away the underlying transport. However, iroh intentionally exposes QUIC's powerful stream API because: + +1. **QUIC is more expressive than TCP** - Multiple concurrent streams, fine-grained flow control, and cancellation give you tools TCP never had +2. **Protocol design matters** - How you structure requests, responses, and notifications affects performance, memory usage, and user experience +3. **No one-size-fits-all** - A file transfer protocol needs different patterns than a chat app or real-time collaboration tool + +Think of iroh as giving you **reliable, secure tunnels between peers**. This guide shows you how to use QUIC's streaming patterns to build efficient protocols inside those tunnels. Whether you're adapting an existing protocol or designing something new, understanding these patterns will help you make the most of iroh's capabilities. + + +iroh uses [noq](https://github.com/n0-computer/noq), a pure-Rust QUIC implementation maintained by [n0.computer](https://n0.computer). noq is production-ready, actively maintained, and used by projects beyond iroh. If you need lower-level QUIC access or want to understand the implementation details, check out the [noq repository](https://github.com/n0-computer/noq). + + + +### Disabling QUIC To use a custom transport exclusively and disable QUIC over IP: @@ -40,3 +52,481 @@ let endpoint = Endpoint::builder() Custom transport support requires the `unstable-custom-transports` feature flag. The API is unstable and subject to change. See [PR #3845](https://github.com/n0-computer/iroh/pull/3845) for background. + +### Overview of the QUIC API + +Implementing a new protocol on the QUIC protocol can be a little daunting initially. Although the API is not that extensive, it's more complicated than e.g. TCP where you can only send and receive bytes, and eventually have an end of stream. +There isn't "one right way" to use the QUIC API. It depends on what interaction pattern your protocol intends to use. +This document is an attempt at categorizing the interaction patterns. Perhaps you find exactly what you want to do here. If not, perhaps the examples give you an idea for how you can utilize the QUIC API for your use case. +One thing to point out is that we're looking at interaction patterns *after* establishing a connection, i.e. everything that happens *after* we've `connect`ed or `accept`ed incoming connections, so everything that happens once we have a `Connection` instance. + + +Unlike TCP, in QUIC you can open multiple streams. Either side of a connection can decide to "open" a stream at any time: +```rs +impl Connection { + async fn open_uni(&self) -> Result; + async fn accept_uni(&self) -> Result, ConnectionError>; + + async fn open_bi(&self) -> Result<(SendStream, RecvStream), ConnectionError>; + async fn accept_bi(&self) -> Result, ConnectionError>; +} +``` +Similar to how each `write` on one side of a TCP-based protocol will correspond to a `read` on the other side, when a protocol `open`s a stream on one end, the other side of the protocol can `accept` such a stream. +Streams can be either uni-directional (`open_uni`/`accept_uni`), or bi-directional (`open_bi`/`accept_bi`). +- With uni-directional streams, only the opening side sends bytes to the accepting side. The receiving side can already start consuming bytes before the opening/sending side finishes writing all data. So it supports streaming, as the name suggests. +- With bi-directional streams, both sides can send bytes to each other at the same time. The API supports full duplex streaming. + + +One bi-directional stream is essentially the closest equivalent to a TCP stream. If your goal is to adapt a TCP protocol to the QUIC API, the easiest way is going to be opening a single bi-directional stream and then essentially using the send and receive stream pair as if it were a TCP stream. + + +Speaking of "finishing writing data", there are some additional ways to communicate information via streams besides sending and receiving bytes! +- The `SendStream` side can `.finish()` the stream. This will send something like an "end of stream notification" to the other side, *after all pending bytes have been sent on the stream.* + This "notification" can be received on the other end in various ways: + - `RecvStream::read` will return `Ok(None)`, if all pending data was read and the stream was finished. Other methods like `read_chunk` work similarly. + - `RecvStream::read_to_end` will resolve once the finishing notification comes in, returning all pending data. **If the sending side never calls `.finish()`, this will never resolve**. + - `RecvStream::stop` will resolve with `Ok(None)` if the stream was finished (or `Ok(Some(code))` if it was reset). +- The `SendStream` side can also `.reset()` the stream. This will have the same effect as `.finish()`ing the stream, except for two differences: + Resetting will happen immediately and discard any pending bytes that haven't been sent yet. You can provide an application-specific "error code" (a `VarInt`) to signal the reason for the reset to the other side. + This "notification" is received in these ways on the other end: + - `RecvStream::read` and other methods like `read_exact`, `read_chunk` and `read_to_end` will return a `ReadError::Reset(code)` with the error code given on the send side. + - `RecvStream::stop` will resolve to the error code `Ok(Some(code))`. +- The other way around, the `RecvStream` side can also notify the sending side that it's not interested in reading any more data by calling `RecvStream::stop` with an application-specific code. + This notification is received on the *sending* side: + - `SendStream::write` and similar methods like `write_all`, `write_chunk` etc. will error out with a `WriteError::Stopped(code)`. + - `SendStream::stopped` resolves with `Ok(code)`. + + +What is the difference between a bi-directional stream and two uni-directional streams? +1. The bi-directional stream establishes the stream pair in a single "open -> accept" interaction. For two uni-directional streams in two directions, you'd need one side to open, then send data, then accept at the same time. The other side would have to accept and then open a stream. +2. Two uni-directional streams can not be stopped or reset as a unit: One stream might be stopped or reset with one close code while the other is still open. Bi-directional streams can only be stopped or reset as a unit. + + +These additional "notification" mechanisms are a common source of confusion: Naively, we might expect a networking API to be able to send and receive bytes, and maybe listen for a "stop". +However, it turns out that with the QUIC API, we can notify the other side about newly opened streams, and finish, reset, or even stop them. Additionally, there's two different types of stream-opening (uni-directional and bi-directional). +A bi-directional stream has 3 different ways *each side* can close *some* aspect of it: Each side can either `.finish()` or `.reset()` its send half, or `.stop()` its receiving half. + +Finally, there's one more important "notification" we have to cover: +Closing the connection. + +Either end of the connection can decide to close the connection at any point by calling `Connection::close` with an application-specific error code (a `VarInt`), (and even a bunch of bytes indicating a "reason", possibly some human-readable ASCII, but without a guarantee that it will be delivered). + +Once this notification is received on the other end, all stream writes return `WriteError::ConnectionLost(ConnectionError::ApplicationClosed { .. })` and all reads return `ReadError::ConnectionLost(ConnectionError::ApplicationClosed { .. })`. +It can also be received by waiting for `Connection::closed` to resolve. + +Importantly, this notification interrupts all flows of data: +- On the side that triggers it, it will drop all data to be sent +- On the side that receives it, it will immediately drop all data to be sent and the side will stop receiving new data. + +What this means is that it's important to carefully close the connection at the right time, either at a point in the protocol where we know that we won't be sending or receiving any more data, or when we're sure we want to interrupt all data flows. + +On the other hand, we want to make sure that we end protocols by sending this notification on at least one end of the connection, as just "leaving the connection hanging" on one endpoint causes the other endpoint to needlessly wait for more information, eventually timing out. + +--- + +Let's look at some interaction pattern examples so we get a feeling for how all of these pieces fit together: + +## Request and Response + +The most common type of protocol interaction. +In this case, the connecting endpoint first sends a request. +The accepting endpoint will read the full request before starting to send a response. +Once the connecting endpoint has read the full response, it will close the connection. +The accepting endpoint waits for this close notification before shutting down. +```rs +async fn connecting_endpoint(conn: Connection, request: &[u8]) -> Result> { + let (mut send, mut recv) = conn.open_bi().await?; + send.write_all(request).await?; + send.finish()?; + + let response = recv.read_to_end(MAX_RESPONSE_SIZE).await?; + + conn.close(0u32.into(), b"I have everything, thanks!"); + + Ok(response) +} + +async fn accepting_endpoint(conn: Connection) -> Result<()> { + let (mut send, mut recv) = conn.accept_bi().await?.ok_or_else(|| anyhow!("connection closed"))?; + let request = recv.read_to_end(MAX_REQUEST_SIZE).await?; + + let response = compute_response(&request); + send.write_all(&response).await?; + send.finish()?; + + conn.closed().await; + + Ok(()) +} +``` + +### Full duplex Request & Response streaming + +It's possible to start sending a response before the request has finished coming in. +This makes it possible to handle arbitrarily big requests in O(1) memory. +In this toy example we're reading `u64`s from the client and send back each of them doubled. + +```rs +async fn connecting_endpoint(conn: Connection, mut request: impl Stream) -> Result<()> { + let (mut send, mut recv) = conn.open_bi().await?; + + // concurrently read the responses + let read_task = tokio::spawn(async move { + let mut buf = [0u8; size_of::()]; + // read_exact will return `Err` once the other side + // finishes its stream + while recv.read_exact(&mut buf).await.is_ok() { + let number = u64::from_be_bytes(buf); + println!("Read response: {number}"); + } + }); + + while let Some(number) = request.next().await { + let bytes = number.to_be_bytes(); + send.write_all(&bytes).await?; + } + send.finish()?; + + // we close the connection after having read all data + read_task.await?; + conn.close(0u32.into(), b"done"); + + Ok(()) +} + +async fn accepting_endpoint(conn: Connection) -> Result<()> { + let (mut send, mut recv) = conn.accept_bi().await?.ok_or_else(|| anyhow!("connection closed"))?; + + let mut buf = [0u8; size_of::()]; + while recv.read_exact(&mut buf).await.is_ok() { + let number = u64::from_be_bytes(buf); + let doubled = number.wrapping_mul(2).to_be_bytes(); + send.write_all(&doubled).await?; + } + send.finish()?; + + // the other side will tell us when it's done reading our data + conn.closed().await; + + Ok(()) +} +``` + +### Multiple Requests & Responses + +This is one of the main use cases QUIC was designed for: Multiplex multiple requests and responses on the same connection. +HTTP3 is an example for a protocol using QUIC's capabilities for this. +A single HTTP3 connection to a server can handle multiple HTTP requests concurrently without the requests blocking each other. +This is the main innovation in HTTP3: It makes HTTP/2's connection pool obsolete. + +In HTTP3, each HTTP request is run as its own bi-directional stream. The request is sent in one direction while the response is received in the other direction. This way both stream directions are cancellable as a unit, this makes it possible for the user agent to cancel some HTTP requests without cancelling any others in the same HTTP3 connection. + +Using the QUIC API for this purpose will feel very natural: +```rs +// The connecting endpoint can call this multiple times +// for one connection. +// When it doesn't want to do more requests and has all +// responses, it can close the connection. +async fn request(conn: &Connection, request: &[u8]) -> Result> { + let (mut send, mut recv) = conn.open_bi().await?; + send.write_all(request).await?; + send.finish()?; + + let response = recv.read_to_end(MAX_RESPONSE_SIZE).await?; + + Ok(response) +} + +// The accepting endpoint will call this to handle all +// incoming requests on a single connection. +async fn handle_requests(conn: Connection) -> Result<()> { + loop { + let stream = conn.accept_bi().await?; + match stream { + Some((send, recv)) => { + tokio::spawn(handle_request(send, recv)); + } + None => break, // connection closed + } + } + Ok(()) +} + +async fn handle_request(mut send: SendStream, mut recv: RecvStream) -> Result<()> { + let request = recv.read_to_end(MAX_REQUEST_SIZE).await?; + + let response = compute_response(&request); + send.write_all(&response).await?; + send.finish()?; + + Ok(()) +} +``` + +Please note that, in this case, the client doesn't immediately close the connection after a single request (duh!). Instead, it might want to optimistically keep the connection open for some idle time or until it knows the application won't need to make another request, and only then close the connection. All that said, it's still true that **the connecting side closes the connection**. + +### Multiple ordered Notifications + +Sending and receiving multiple notifications that can be handled one-by-one can be done by adding framing to the bytes on a uni-directional stream. + +```rs +async fn connecting_endpoint(conn: Connection, mut notifications: impl Stream + Unpin) -> Result<()> { + let mut send = conn.open_uni().await?; + + let mut send_frame = LengthDelimitedCodec::builder().new_write(send); + while let Some(notification) = notifications.next().await { + send_frame.send(notification).await?; + } + + send_frame.get_mut().finish()?; + conn.closed().await; + + Ok(()) +} + +async fn accepting_endpoint(conn: Connection) -> Result<()> { + let recv = conn.accept_uni().await?.ok_or_else(|| anyhow!("connection closed"))?; + let mut recv_frame = LengthDelimitedCodec::builder().new_read(recv); + + while let Some(notification) = recv_frame.try_next().await? { + println!("Received notification: {notification:?}"); + } + + conn.close(0u32.into(), b"got everything!"); + + Ok(()) +} +``` + +Here we're using `LengthDelimitedCodec` and `tokio-util`'s `codec` feature to easily turn the `SendStream` and `RecvStream` that work as streams of bytes into streams of items, where each item in this case is a `Bytes`/`BytesMut`. In practice you would probably add byte parsing to this code first, and you might want to configure the `LengthDelimitedCodec`. + +The resulting notifications are all in order since the bytes in the uni-directional streams are received in-order, and we're processing one frame before continuing to read the next bytes off of the QUIC stream. + + +There's another somewhat common way of doing this: +The order that `accept_uni` come in will match the order that `open_uni` are called on the remote endpoint. (The same also goes for bi-directional streams.) +This way you would receive one notification per stream and know the order of notifications from the stream ID/the order of accepted streams. +The downside of doing it that way is you will occupy more than one stream. If you want to multiplex other things on the same connection, you'll need to add some signaling. + + + +### Request with multiple Responses + +If your protocol expects multiple responses for a request, we can implement that with the same primitive we've learned about in the section about multiple ordered notifications: We use framing to segment a single response byte stream into multiple ordered responses: + +```rs +async fn connecting_endpoint(conn: Connection, request: &[u8]) -> Result<()> { + let (mut send, recv) = conn.open_bi().await?; + send.write_all(request).await?; + send.finish()?; + + let mut recv_frame = LengthDelimitedCodec::builder().new_read(recv); + while let Some(response) = recv_frame.try_next().await? { + println!("Received response: {response:?}"); + } + + conn.close(0u32.into(), b"thank you!"); + + Ok(()) +} + +async fn accepting_endpoint(conn: Connection) -> Result<()> { + let (send, mut recv) = conn.accept_bi().await?.ok_or_else(|| anyhow!("connection closed"))?; + let request = recv.read_to_end(MAX_REQUEST_SIZE).await?; + + let mut send_frame = LengthDelimitedCodec::builder().new_write(send); + let mut responses = responses_for_request(&request); + while let Some(response) = responses.next().await { + send_frame.send(response).await?; + } + send_frame.get_mut().finish()?; + + conn.closed().await; + + Ok(()) +} + +fn responses_for_request(req: &[u8]) -> impl Stream { + // ... +} +``` + +This example ends up similar as the one with ordered notifications, except +1. The roles are reversed: The length-prefix sending happens on the accepting endpoint, and the length-prefix decoding on the connecting endpoint. +2. We additionally send a request before we start receiving multiple responses. + + +At this point you should have a good feel for how to write request/response protocols using the QUIC API. For example, you should be able to piece together a full-duplex request/response protocol where you're sending the request as multiple frames and the response comes in with multiple frames, too, by combining two length delimited codes in both ways and taking notes from the full duplex section further above. + + +### Requests with multiple unordered Responses + +The previous example required all responses to come in ordered. +What if that's undesired? What if we want the connecting endpoint to receive incoming responses as quickly as possible? +In that case, we need to break up the single response stream into multiple response streams. +We can do this by "conceptually" splitting the "single" bi-directional stream into one uni-directional stream for the request and multiple uni-directional streams in the other direction for all the responses: + +```rs +async fn connecting_side(conn: Connection, request: &[u8]) -> Result<()> { + let mut send = conn.open_uni().await?; + send.write_all(request).await?; + send.finish()?; + + let recv_tasks = TaskTracker::new(); + // accept_uni will return `Ok(None)` once the connection is closed + loop { + match conn.accept_uni().await? { + Some(recv) => { + recv_tasks.spawn(handle_response(recv)); + } + None => break, + } + } + recv_tasks.wait().await; + conn.close(0u32.into(), b"Thank you!"); + + Ok(()) +} +``` + + +You might've noticed that this destroys the "association" between the two stream directions. This means we can't use tricks similar to what HTTP3 does that we described above to multiplex multiple request-responses interactions on the same connection. +This is unfortunate, but can be fixed by prefixing your requests and responses with a unique ID chosen per request. This ID then helps associate the responses to the requests that used the same ID. +Another thing that might or might not be important for your use case is knowing when unordered stream of responses is "done": +You can either introduce another message type that is interpreted as a finishing token, but there's another elegant way of solving this. Instead of only opening a uni-directional stream for the request, you open a bi-directional one. The response stream will only be used to indicate the final response stream ID. It then acts as a sort of "control stream" to provide auxiliary information about the request for the connecting endpoint. + + +## Time-sensitive Real-time interaction + +We often see users reaching for the QUIC datagram extension when implementing real-time protocols. Doing this is in most cases misguided. +QUIC datagram sending still interacts with QUIC's congestion controller and thus are also acknowledged. +Implementing traditional protocols on top of QUIC datagrams might thus not perform the way they were designed to. +Instead, it's often better to use lots of streams that are then stopped, reset or prioritized. + +A real-world example is the [media over QUIC protocol](https://doc.moq.dev/) (MoQ in short): MoQ is used +to transfer live video frames. It uses one QUIC stream for each frame (QUIC +streams are cheap to create)! + +The receiver then stops streams that are "too old" to be delivered, e.g. because it's a live video stream and newer frames were already fully received. +Similarly, the sending side will also reset older streams for the application level to indicate to the QUIC stack it doesn't need to keep re-trying the transmission of an outdated live video frame. (MoQ will actually also use stream prioritization to make sure the newest video frames get scheduled to be sent first.) + +## Closing Connections + +Gracefully closing connections can be tricky to get right when first working with the QUIC API. +If you don't close connections gracefully, you'll see the connecting timing out on one endpoint, usually after 30s, even though another endpoint finishes promptly without errors. +This happens when the endpoint that finishes doesn't notify the other endpoint about having finished operations. +There's mainly two reasons this happens: +1. The protocol doesn't call `Connection::close` at the right moment. +2. The endpoint that closes the connection is immediately dropped afterwards without waiting for `Endpoint::close`. +To make sure that you're not hitting (2), simply always make sure to wait for `Endpoint::close` to resolve, on both `Endpoint`s, if you can afford it. +Getting (1) right is harder. We might accidentally close connections too early, because we accidentally drop the `Connection` (which implicitly calls close). Instead, we should always keep around the connection and either wait for `Connection::closed` to resolve or call `Connection::close` ourselves at the right moment. When that is depends on what kind of protocol you're implementing: +### After a single Interaction + +Protocols that implement a single interaction want to keep their connection alive for only the time of this interaction. +In this case, the endpoint that received application data last will be the endpoint that calls `Connection::close` at that point in time. +Conversely, the other endpoint should wait for `Connection::closed` to resolve before ending its operations. +An example of this can be seen in the [Request and Response](#request-and-response) section above: The connecting side closes the connection once it received the response and the accepting side waits for the connection to be closed after having sent off the response. + +### During continuous Interaction + +Sometimes we want to keep open connections as long as the user is actively working with the application, so we don't needlessly run handshakes or try to hole-punch repeatedly. +In these cases, the protocol flow doesn't indicate which endpoint of the connection will be the one that closes the connection. +Instead, clients should concurrently monitor `Connection::closed` while they're running the protocol: + +```rs +async fn handle_connection(conn: Connection) -> Result<()> { + futures_lite::future::race( + run_protocol(conn.clone()), + async move { + conn.closed().await; + anyhow::Ok(()) + }, + ).await?; + Ok(()) +} + +async fn run_protocol(conn: Connection) -> Result<()> { + // run normal protocol flow + // once we realize we want to abort the connection flow + conn.close(0u32.into(), b"ah sorry, have to go!"); + + Ok(()) +} +``` + +And again, after `handle_connection` we need to make sure to wait for `Endpoint::close` to resolve. + +## Aborting Streams + +Sometimes you need to abandon a stream before it completes - either because the data has become stale, or because you've decided you no longer need it. QUIC provides mechanisms for both the sender and receiver to abort streams gracefully. + +### When to abort streams + +A real-world example comes from Media over QUIC (MoQ), which streams live video frames. Consider this scenario: + +- Each video frame is sent on its own uni-directional stream +- Frames arrive out of order due to network conditions +- By the time an old frame finishes transmitting, newer frames have already been received +- Continuing to receive the old frame wastes bandwidth and processing time + +### How to abort streams + +**From the sender side:** + +Use `SendStream::reset(error_code)` to immediately stop sending data and discard any buffered bytes. This tells QUIC to stop retrying lost packets for this stream. + + +**From the receiver side:** + +Use `RecvStream::stop(error_code)` to tell the sender you're no longer interested in the data. This allows the sender's QUIC stack to stop retransmitting lost packets. + + +### Key insights + +1. **Stream IDs indicate order**: QUIC stream IDs are monotonically increasing. You can compare stream IDs to determine which streams are newer without relying on application-level sequencing. + +2. **Both sides can abort**: Either the sender (via `reset`) or receiver (via `stop`) can abort a stream. Whichever side detects the data is no longer needed first should initiate the abort. + +3. **QUIC stops retransmissions**: When a stream is reset or stopped, QUIC immediately stops trying to recover lost packets for that stream, saving bandwidth and processing time. + +4. **Streams are cheap**: Opening a new stream is very fast (no round-trips required), so it's perfectly fine to open one stream per video frame, message, or other small unit of data. + +This pattern of using many short-lived streams that can be individually aborted is one of QUIC's most powerful features for real-time applications. It gives you fine-grained control over what data is worth transmitting, without the head-of-line blocking issues that would occur with a single TCP connection. + +## QUIC 0-RTT features + +### Server-side 0.5-RTT + +QUIC connections always take 1 full round-trip time (RTT) to establish - the client sends a hello, the server responds with its hello and certificate, and only then can application data flow. However, the server can actually start sending application data **before** the client finishes the handshake, achieving what's called "0.5-RTT" latency. + +This works because after the server sends its hello, it doesn't need to wait for the client's final handshake message before it can start processing the client's request and sending a response. The server knows the encryption keys at this point and can immediately begin sending data back. + +**How to use it:** + +On the server side, this happens automatically - you don't need to do anything special. As soon as you `accept_bi()` or `accept_uni()` a stream, you can start writing to it immediately, even if the handshake hasn't fully completed on the client side yet. + +```rs +async fn accepting_endpoint(conn: Connection) -> Result<()> { + let (mut send, mut recv) = conn.accept_bi().await?.ok_or_else(|| anyhow!("connection closed"))?; + let request = recv.read_to_end(MAX_REQUEST_SIZE).await?; + + // We can start sending the response immediately without waiting + // for the client to finish the handshake + let response = compute_response(&request); + send.write_all(&response).await?; + send.finish()?; + + conn.closed().await; + Ok(()) +} +``` + +**Important gotcha: Request replay attacks** + +Because the server starts processing before the handshake completes, there's a security consideration: the client's initial request data could potentially be replayed by an attacker who intercepts the handshake packets. This means: + +- **The server should treat 0.5-RTT requests as potentially non-idempotent** +- Avoid performing actions with side effects (like making payments, deleting data, etc.) based solely on 0.5-RTT data +- If your protocol requires idempotency guarantees, wait for the handshake to complete before processing sensitive operations + +For read-only operations or idempotent requests, 0.5-RTT is perfectly safe and provides a nice latency improvement. From acd4948488f0a91fc3d071a81234ad27d9379fda Mon Sep 17 00:00:00 2001 From: Rae McKelvey <633012+okdistribute@users.noreply.github.com> Date: Tue, 17 Mar 2026 10:45:26 -0700 Subject: [PATCH 4/7] move transports down --- docs.json | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs.json b/docs.json index 3497d96..0435a45 100644 --- a/docs.json +++ b/docs.json @@ -44,16 +44,6 @@ "connecting/endpoint-hooks" ] }, - { - "group": "Transports", - "pages": [ - "transports/quic", - "transports/mdns", - "transports/tor", - "transports/nym", - "transports/bluetooth" - ] - }, { "group": "Protocols", "pages": [ @@ -66,6 +56,16 @@ "protocols/writing-a-protocol" ] }, + { + "group": "Transports", + "pages": [ + "transports/quic", + "transports/mdns", + "transports/tor", + "transports/nym", + "transports/bluetooth" + ] + }, { "group": "Deployment", "pages": [ From e049fe676e0cb19196db0e6e1730b166768d1e1f Mon Sep 17 00:00:00 2001 From: rae <633012+okdistribute@users.noreply.github.com> Date: Wed, 18 Mar 2026 10:53:28 -0700 Subject: [PATCH 5/7] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- connecting/creating-endpoint.mdx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/connecting/creating-endpoint.mdx b/connecting/creating-endpoint.mdx index 09c134c..26d07c6 100644 --- a/connecting/creating-endpoint.mdx +++ b/connecting/creating-endpoint.mdx @@ -16,11 +16,13 @@ to listen for incoming connections. ```rust use iroh::{Endpoint, presets}; +use anyhow::Result; #[tokio::main] -async fn main() { +async fn main() -> Result<()> { let endpoint = Endpoint::bind(presets::N0).await?; // ... + Ok(()) } ``` From 7be9c575f14ab0ab2876c934a3b2650939750cd0 Mon Sep 17 00:00:00 2001 From: rae <633012+okdistribute@users.noreply.github.com> Date: Wed, 18 Mar 2026 10:53:43 -0700 Subject: [PATCH 6/7] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- quickstart.mdx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/quickstart.mdx b/quickstart.mdx index e65c805..83b8a29 100644 --- a/quickstart.mdx +++ b/quickstart.mdx @@ -37,6 +37,8 @@ connection to the closest relay, and finds ways to address devices by `EndpointId`. ```rust +use iroh::{Endpoint, presets}; + #[tokio::main] async fn main() -> anyhow::Result<()> { // Create an endpoint, it allows creating and accepting From 49ffbb8ebbb6c34cd1e8890288799fe612510b57 Mon Sep 17 00:00:00 2001 From: Rae McKelvey <633012+okdistribute@users.noreply.github.com> Date: Wed, 18 Mar 2026 11:20:23 -0700 Subject: [PATCH 7/7] comments from phillip --- connecting/local-discovery.mdx | 39 +++ docs.json | 8 +- protocols/using-quic.md | 21 +- protocols/writing-a-protocol.mdx | 2 +- transports/mdns.mdx | 39 --- transports/quic.mdx | 532 ------------------------------- 6 files changed, 63 insertions(+), 578 deletions(-) create mode 100644 connecting/local-discovery.mdx delete mode 100644 transports/mdns.mdx delete mode 100644 transports/quic.mdx diff --git a/connecting/local-discovery.mdx b/connecting/local-discovery.mdx new file mode 100644 index 0000000..7c130c7 --- /dev/null +++ b/connecting/local-discovery.mdx @@ -0,0 +1,39 @@ +--- +title: "mDNS" +--- + +The mDNS discovery mechanism will automatically broadcast your endpoint's +presence on the local network, and listen for other endpoints doing the same. When +another endpoint is discovered, the dialing information is exchanged, and a +connection can be established directly over the local network without needing a relay. + +Devices need to be connected to the same local network for mDNS discovery to +work. This can be a Wi-Fi network, an Ethernet network, or even a mobile +hotspot. mDNS is not designed to work over the internet or across different +networks. + +## Usage + +Local Discovery is _not_ enabled by default, and must be enabled explicitly. +You'll need to add the `discovery-local-network` feature flag to your +`Cargo.toml` to use it. + +```toml +[dependencies] +# Make sure to use the most recent version here instead of nn. (at the time of writing: 0.32) +iroh = { version = "0.nn", features = ["address-lookup-mdns"] } +``` + +Then configure your endpoint to use local discovery concurrently with the default DNS discovery: + +```rust +use iroh::{Endpoint, presets}; + +let mdns = iroh::address_lookup::mdns::MdnsAddressLookup::builder(); +let ep = Endpoint::builder() + .address_lookup(mdns) + .bind(presets::N0) + .await?; +``` + +For more information on how mDNS discovery works, see the [mDNS documentation](https://docs.rs/iroh/latest/iroh/address_lookup/mdns/index.html). diff --git a/docs.json b/docs.json index 0435a45..a190daf 100644 --- a/docs.json +++ b/docs.json @@ -40,12 +40,13 @@ "connecting/custom-relays", "connecting/dns-discovery", "connecting/dht-discovery", + "connecting/local-discovery", "connecting/gossip", "connecting/endpoint-hooks" ] }, { - "group": "Protocols", + "group": "Sending Data", "pages": [ "protocols/kv-crdts", "protocols/blobs", @@ -53,14 +54,13 @@ "protocols/automerge", "protocols/streaming", "examples/chat", - "protocols/writing-a-protocol" + "protocols/writing-a-protocol", + "protocols/using-quic" ] }, { "group": "Transports", "pages": [ - "transports/quic", - "transports/mdns", "transports/tor", "transports/nym", "transports/bluetooth" diff --git a/protocols/using-quic.md b/protocols/using-quic.md index 9a70209..fdd7578 100644 --- a/protocols/using-quic.md +++ b/protocols/using-quic.md @@ -2,9 +2,26 @@ title: "Using QUIC" --- -## Why this matters for iroh +Every endpoint uses QUIC over UDP by default — no configuration required. -iroh is built on top of QUIC, providing connectivity, NAT traversal, and encrypted connections out of the box. While iroh handles the hard parts of networking—holepunching, relay servers, and discovery—**you still need to design how your application exchanges data once connected**. +iroh's QUIC implementation is built on +[noq](https://github.com/n0-computer/noq), which includes multipath support and +QUIC NAT traversal. + +All connections are encrypted and authenticated using TLS 1.3. Holepunching, +relay fallback, and multipath are all handled at the QUIC layer automatically. + +## Custom transports + +QUIC over UDP is the default, but iroh supports plugging in additional custom +transports alongside it. + +All transports, even custom transports [Tor](/transports/tor), [Nym](/transports/nym), and +[Bluetooth](/transports/bluetooth) deliver QUIC datagrams. + +## Using QUIC + +While iroh handles the hard parts of networking—holepunching, relay servers, and discovery—**you still need to design how your application exchanges data once connected**. Many developers reach for iroh expecting it to completely abstract away the underlying transport. However, iroh intentionally exposes QUIC's powerful stream API because: diff --git a/protocols/writing-a-protocol.mdx b/protocols/writing-a-protocol.mdx index 2b3cab8..1f00720 100644 --- a/protocols/writing-a-protocol.mdx +++ b/protocols/writing-a-protocol.mdx @@ -164,7 +164,7 @@ This follows the [request-response pattern](/protocols/using-quic#request-and-re ```rs async fn connect_side(addr: EndpointAddr) -> Result<()> { - let endpoint = Endpoint::bind(presets::N0).await?; + let endpoint = Endpoint::bind(iroh::presets::N0).await?; // Open a connection to the accepting endpoint let conn = endpoint.connect(addr, ALPN).await?; diff --git a/transports/mdns.mdx b/transports/mdns.mdx deleted file mode 100644 index 22aa823..0000000 --- a/transports/mdns.mdx +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: "mDNS" ---- - -mDNS (Multicast DNS) lets iroh endpoints discover and connect to each other on a local network without needing an internet connection or relay server. - -Local connections can be faster and more reliable than internet-based connections, especially in environments with poor connectivity. They also enhance privacy by keeping communications within a local area. - -Local discovery is _not_ enabled by default and must be opted into. - -## Installation - -Add the `address-lookup-mdns` feature flag: - -```toml -[dependencies] -iroh = { version = "0.nn", features = ["address-lookup-mdns"] } -``` - -## Usage - -Configure your endpoint to use mDNS alongside the default DNS address lookup: - -```rust -use iroh::{Endpoint, presets}; - -let mdns = iroh::address_lookup::mdns::MdnsAddressLookup::builder(); -let endpoint = Endpoint::builder() - .address_lookup(mdns) - .bind(presets::N0) - .await?; -``` - -The mDNS mechanism automatically broadcasts your endpoint's presence on the -local network and listens for other endpoints doing the same. When another -endpoint is discovered, dialing information is exchanged and a connection can be -established directly over the local network: no relay needed. - -For more information see the [mDNS API docs](https://docs.rs/iroh/latest/iroh/address_lookup/mdns/index.html). diff --git a/transports/quic.mdx b/transports/quic.mdx deleted file mode 100644 index 7e28089..0000000 --- a/transports/quic.mdx +++ /dev/null @@ -1,532 +0,0 @@ ---- -title: "QUIC" ---- - -QUIC is the default transport in iroh. Every endpoint uses QUIC over UDP by default — no configuration required. - -iroh's QUIC implementation is built on [noq](https://github.com/n0-computer/noq), n0's fork of the Quinn QUIC library. Over time the fork diverged significantly as n0 added multipath support, QUIC NAT traversal, and other iroh-specific features. - -All connections are encrypted and authenticated using TLS 1.3. Hole-punching, relay fallback, and multi-path are all handled at the QUIC layer automatically. - -For details on how to use QUIC streams to build protocols on top of an iroh connection, see [Using QUIC](/protocols/using-quic). - -## Custom transports - -QUIC over UDP is the default, but iroh supports plugging in additional custom -transports alongside it. See [Tor](/transports/tor), [Nym](/transports/nym), and -[Bluetooth](/transports/bluetooth) for examples. - -## Using QUIC - -### Why this matters for iroh - -iroh is built on top of QUIC, providing connectivity, NAT traversal, and encrypted connections out of the box. While iroh handles the hard parts of networking—holepunching, relay servers, and discovery—**you still need to design how your application exchanges data once connected**. - -Many developers reach for iroh expecting it to completely abstract away the underlying transport. However, iroh intentionally exposes QUIC's powerful stream API because: - -1. **QUIC is more expressive than TCP** - Multiple concurrent streams, fine-grained flow control, and cancellation give you tools TCP never had -2. **Protocol design matters** - How you structure requests, responses, and notifications affects performance, memory usage, and user experience -3. **No one-size-fits-all** - A file transfer protocol needs different patterns than a chat app or real-time collaboration tool - -Think of iroh as giving you **reliable, secure tunnels between peers**. This guide shows you how to use QUIC's streaming patterns to build efficient protocols inside those tunnels. Whether you're adapting an existing protocol or designing something new, understanding these patterns will help you make the most of iroh's capabilities. - - -iroh uses [noq](https://github.com/n0-computer/noq), a pure-Rust QUIC implementation maintained by [n0.computer](https://n0.computer). noq is production-ready, actively maintained, and used by projects beyond iroh. If you need lower-level QUIC access or want to understand the implementation details, check out the [noq repository](https://github.com/n0-computer/noq). - - - -### Disabling QUIC - -To use a custom transport exclusively and disable QUIC over IP: - -```rust -use iroh::presets; - -let endpoint = Endpoint::builder() - .clear_ip_transports() - .add_custom_transport(my_transport) - .bind(presets::N0) - .await?; -``` - - -Custom transport support requires the `unstable-custom-transports` feature flag. The API is unstable and subject to change. See [PR #3845](https://github.com/n0-computer/iroh/pull/3845) for background. - - -### Overview of the QUIC API - -Implementing a new protocol on the QUIC protocol can be a little daunting initially. Although the API is not that extensive, it's more complicated than e.g. TCP where you can only send and receive bytes, and eventually have an end of stream. -There isn't "one right way" to use the QUIC API. It depends on what interaction pattern your protocol intends to use. -This document is an attempt at categorizing the interaction patterns. Perhaps you find exactly what you want to do here. If not, perhaps the examples give you an idea for how you can utilize the QUIC API for your use case. -One thing to point out is that we're looking at interaction patterns *after* establishing a connection, i.e. everything that happens *after* we've `connect`ed or `accept`ed incoming connections, so everything that happens once we have a `Connection` instance. - - -Unlike TCP, in QUIC you can open multiple streams. Either side of a connection can decide to "open" a stream at any time: -```rs -impl Connection { - async fn open_uni(&self) -> Result; - async fn accept_uni(&self) -> Result, ConnectionError>; - - async fn open_bi(&self) -> Result<(SendStream, RecvStream), ConnectionError>; - async fn accept_bi(&self) -> Result, ConnectionError>; -} -``` -Similar to how each `write` on one side of a TCP-based protocol will correspond to a `read` on the other side, when a protocol `open`s a stream on one end, the other side of the protocol can `accept` such a stream. -Streams can be either uni-directional (`open_uni`/`accept_uni`), or bi-directional (`open_bi`/`accept_bi`). -- With uni-directional streams, only the opening side sends bytes to the accepting side. The receiving side can already start consuming bytes before the opening/sending side finishes writing all data. So it supports streaming, as the name suggests. -- With bi-directional streams, both sides can send bytes to each other at the same time. The API supports full duplex streaming. - - -One bi-directional stream is essentially the closest equivalent to a TCP stream. If your goal is to adapt a TCP protocol to the QUIC API, the easiest way is going to be opening a single bi-directional stream and then essentially using the send and receive stream pair as if it were a TCP stream. - - -Speaking of "finishing writing data", there are some additional ways to communicate information via streams besides sending and receiving bytes! -- The `SendStream` side can `.finish()` the stream. This will send something like an "end of stream notification" to the other side, *after all pending bytes have been sent on the stream.* - This "notification" can be received on the other end in various ways: - - `RecvStream::read` will return `Ok(None)`, if all pending data was read and the stream was finished. Other methods like `read_chunk` work similarly. - - `RecvStream::read_to_end` will resolve once the finishing notification comes in, returning all pending data. **If the sending side never calls `.finish()`, this will never resolve**. - - `RecvStream::stop` will resolve with `Ok(None)` if the stream was finished (or `Ok(Some(code))` if it was reset). -- The `SendStream` side can also `.reset()` the stream. This will have the same effect as `.finish()`ing the stream, except for two differences: - Resetting will happen immediately and discard any pending bytes that haven't been sent yet. You can provide an application-specific "error code" (a `VarInt`) to signal the reason for the reset to the other side. - This "notification" is received in these ways on the other end: - - `RecvStream::read` and other methods like `read_exact`, `read_chunk` and `read_to_end` will return a `ReadError::Reset(code)` with the error code given on the send side. - - `RecvStream::stop` will resolve to the error code `Ok(Some(code))`. -- The other way around, the `RecvStream` side can also notify the sending side that it's not interested in reading any more data by calling `RecvStream::stop` with an application-specific code. - This notification is received on the *sending* side: - - `SendStream::write` and similar methods like `write_all`, `write_chunk` etc. will error out with a `WriteError::Stopped(code)`. - - `SendStream::stopped` resolves with `Ok(code)`. - - -What is the difference between a bi-directional stream and two uni-directional streams? -1. The bi-directional stream establishes the stream pair in a single "open -> accept" interaction. For two uni-directional streams in two directions, you'd need one side to open, then send data, then accept at the same time. The other side would have to accept and then open a stream. -2. Two uni-directional streams can not be stopped or reset as a unit: One stream might be stopped or reset with one close code while the other is still open. Bi-directional streams can only be stopped or reset as a unit. - - -These additional "notification" mechanisms are a common source of confusion: Naively, we might expect a networking API to be able to send and receive bytes, and maybe listen for a "stop". -However, it turns out that with the QUIC API, we can notify the other side about newly opened streams, and finish, reset, or even stop them. Additionally, there's two different types of stream-opening (uni-directional and bi-directional). -A bi-directional stream has 3 different ways *each side* can close *some* aspect of it: Each side can either `.finish()` or `.reset()` its send half, or `.stop()` its receiving half. - -Finally, there's one more important "notification" we have to cover: -Closing the connection. - -Either end of the connection can decide to close the connection at any point by calling `Connection::close` with an application-specific error code (a `VarInt`), (and even a bunch of bytes indicating a "reason", possibly some human-readable ASCII, but without a guarantee that it will be delivered). - -Once this notification is received on the other end, all stream writes return `WriteError::ConnectionLost(ConnectionError::ApplicationClosed { .. })` and all reads return `ReadError::ConnectionLost(ConnectionError::ApplicationClosed { .. })`. -It can also be received by waiting for `Connection::closed` to resolve. - -Importantly, this notification interrupts all flows of data: -- On the side that triggers it, it will drop all data to be sent -- On the side that receives it, it will immediately drop all data to be sent and the side will stop receiving new data. - -What this means is that it's important to carefully close the connection at the right time, either at a point in the protocol where we know that we won't be sending or receiving any more data, or when we're sure we want to interrupt all data flows. - -On the other hand, we want to make sure that we end protocols by sending this notification on at least one end of the connection, as just "leaving the connection hanging" on one endpoint causes the other endpoint to needlessly wait for more information, eventually timing out. - ---- - -Let's look at some interaction pattern examples so we get a feeling for how all of these pieces fit together: - -## Request and Response - -The most common type of protocol interaction. -In this case, the connecting endpoint first sends a request. -The accepting endpoint will read the full request before starting to send a response. -Once the connecting endpoint has read the full response, it will close the connection. -The accepting endpoint waits for this close notification before shutting down. -```rs -async fn connecting_endpoint(conn: Connection, request: &[u8]) -> Result> { - let (mut send, mut recv) = conn.open_bi().await?; - send.write_all(request).await?; - send.finish()?; - - let response = recv.read_to_end(MAX_RESPONSE_SIZE).await?; - - conn.close(0u32.into(), b"I have everything, thanks!"); - - Ok(response) -} - -async fn accepting_endpoint(conn: Connection) -> Result<()> { - let (mut send, mut recv) = conn.accept_bi().await?.ok_or_else(|| anyhow!("connection closed"))?; - let request = recv.read_to_end(MAX_REQUEST_SIZE).await?; - - let response = compute_response(&request); - send.write_all(&response).await?; - send.finish()?; - - conn.closed().await; - - Ok(()) -} -``` - -### Full duplex Request & Response streaming - -It's possible to start sending a response before the request has finished coming in. -This makes it possible to handle arbitrarily big requests in O(1) memory. -In this toy example we're reading `u64`s from the client and send back each of them doubled. - -```rs -async fn connecting_endpoint(conn: Connection, mut request: impl Stream) -> Result<()> { - let (mut send, mut recv) = conn.open_bi().await?; - - // concurrently read the responses - let read_task = tokio::spawn(async move { - let mut buf = [0u8; size_of::()]; - // read_exact will return `Err` once the other side - // finishes its stream - while recv.read_exact(&mut buf).await.is_ok() { - let number = u64::from_be_bytes(buf); - println!("Read response: {number}"); - } - }); - - while let Some(number) = request.next().await { - let bytes = number.to_be_bytes(); - send.write_all(&bytes).await?; - } - send.finish()?; - - // we close the connection after having read all data - read_task.await?; - conn.close(0u32.into(), b"done"); - - Ok(()) -} - -async fn accepting_endpoint(conn: Connection) -> Result<()> { - let (mut send, mut recv) = conn.accept_bi().await?.ok_or_else(|| anyhow!("connection closed"))?; - - let mut buf = [0u8; size_of::()]; - while recv.read_exact(&mut buf).await.is_ok() { - let number = u64::from_be_bytes(buf); - let doubled = number.wrapping_mul(2).to_be_bytes(); - send.write_all(&doubled).await?; - } - send.finish()?; - - // the other side will tell us when it's done reading our data - conn.closed().await; - - Ok(()) -} -``` - -### Multiple Requests & Responses - -This is one of the main use cases QUIC was designed for: Multiplex multiple requests and responses on the same connection. -HTTP3 is an example for a protocol using QUIC's capabilities for this. -A single HTTP3 connection to a server can handle multiple HTTP requests concurrently without the requests blocking each other. -This is the main innovation in HTTP3: It makes HTTP/2's connection pool obsolete. - -In HTTP3, each HTTP request is run as its own bi-directional stream. The request is sent in one direction while the response is received in the other direction. This way both stream directions are cancellable as a unit, this makes it possible for the user agent to cancel some HTTP requests without cancelling any others in the same HTTP3 connection. - -Using the QUIC API for this purpose will feel very natural: -```rs -// The connecting endpoint can call this multiple times -// for one connection. -// When it doesn't want to do more requests and has all -// responses, it can close the connection. -async fn request(conn: &Connection, request: &[u8]) -> Result> { - let (mut send, mut recv) = conn.open_bi().await?; - send.write_all(request).await?; - send.finish()?; - - let response = recv.read_to_end(MAX_RESPONSE_SIZE).await?; - - Ok(response) -} - -// The accepting endpoint will call this to handle all -// incoming requests on a single connection. -async fn handle_requests(conn: Connection) -> Result<()> { - loop { - let stream = conn.accept_bi().await?; - match stream { - Some((send, recv)) => { - tokio::spawn(handle_request(send, recv)); - } - None => break, // connection closed - } - } - Ok(()) -} - -async fn handle_request(mut send: SendStream, mut recv: RecvStream) -> Result<()> { - let request = recv.read_to_end(MAX_REQUEST_SIZE).await?; - - let response = compute_response(&request); - send.write_all(&response).await?; - send.finish()?; - - Ok(()) -} -``` - -Please note that, in this case, the client doesn't immediately close the connection after a single request (duh!). Instead, it might want to optimistically keep the connection open for some idle time or until it knows the application won't need to make another request, and only then close the connection. All that said, it's still true that **the connecting side closes the connection**. - -### Multiple ordered Notifications - -Sending and receiving multiple notifications that can be handled one-by-one can be done by adding framing to the bytes on a uni-directional stream. - -```rs -async fn connecting_endpoint(conn: Connection, mut notifications: impl Stream + Unpin) -> Result<()> { - let mut send = conn.open_uni().await?; - - let mut send_frame = LengthDelimitedCodec::builder().new_write(send); - while let Some(notification) = notifications.next().await { - send_frame.send(notification).await?; - } - - send_frame.get_mut().finish()?; - conn.closed().await; - - Ok(()) -} - -async fn accepting_endpoint(conn: Connection) -> Result<()> { - let recv = conn.accept_uni().await?.ok_or_else(|| anyhow!("connection closed"))?; - let mut recv_frame = LengthDelimitedCodec::builder().new_read(recv); - - while let Some(notification) = recv_frame.try_next().await? { - println!("Received notification: {notification:?}"); - } - - conn.close(0u32.into(), b"got everything!"); - - Ok(()) -} -``` - -Here we're using `LengthDelimitedCodec` and `tokio-util`'s `codec` feature to easily turn the `SendStream` and `RecvStream` that work as streams of bytes into streams of items, where each item in this case is a `Bytes`/`BytesMut`. In practice you would probably add byte parsing to this code first, and you might want to configure the `LengthDelimitedCodec`. - -The resulting notifications are all in order since the bytes in the uni-directional streams are received in-order, and we're processing one frame before continuing to read the next bytes off of the QUIC stream. - - -There's another somewhat common way of doing this: -The order that `accept_uni` come in will match the order that `open_uni` are called on the remote endpoint. (The same also goes for bi-directional streams.) -This way you would receive one notification per stream and know the order of notifications from the stream ID/the order of accepted streams. -The downside of doing it that way is you will occupy more than one stream. If you want to multiplex other things on the same connection, you'll need to add some signaling. - - - -### Request with multiple Responses - -If your protocol expects multiple responses for a request, we can implement that with the same primitive we've learned about in the section about multiple ordered notifications: We use framing to segment a single response byte stream into multiple ordered responses: - -```rs -async fn connecting_endpoint(conn: Connection, request: &[u8]) -> Result<()> { - let (mut send, recv) = conn.open_bi().await?; - send.write_all(request).await?; - send.finish()?; - - let mut recv_frame = LengthDelimitedCodec::builder().new_read(recv); - while let Some(response) = recv_frame.try_next().await? { - println!("Received response: {response:?}"); - } - - conn.close(0u32.into(), b"thank you!"); - - Ok(()) -} - -async fn accepting_endpoint(conn: Connection) -> Result<()> { - let (send, mut recv) = conn.accept_bi().await?.ok_or_else(|| anyhow!("connection closed"))?; - let request = recv.read_to_end(MAX_REQUEST_SIZE).await?; - - let mut send_frame = LengthDelimitedCodec::builder().new_write(send); - let mut responses = responses_for_request(&request); - while let Some(response) = responses.next().await { - send_frame.send(response).await?; - } - send_frame.get_mut().finish()?; - - conn.closed().await; - - Ok(()) -} - -fn responses_for_request(req: &[u8]) -> impl Stream { - // ... -} -``` - -This example ends up similar as the one with ordered notifications, except -1. The roles are reversed: The length-prefix sending happens on the accepting endpoint, and the length-prefix decoding on the connecting endpoint. -2. We additionally send a request before we start receiving multiple responses. - - -At this point you should have a good feel for how to write request/response protocols using the QUIC API. For example, you should be able to piece together a full-duplex request/response protocol where you're sending the request as multiple frames and the response comes in with multiple frames, too, by combining two length delimited codes in both ways and taking notes from the full duplex section further above. - - -### Requests with multiple unordered Responses - -The previous example required all responses to come in ordered. -What if that's undesired? What if we want the connecting endpoint to receive incoming responses as quickly as possible? -In that case, we need to break up the single response stream into multiple response streams. -We can do this by "conceptually" splitting the "single" bi-directional stream into one uni-directional stream for the request and multiple uni-directional streams in the other direction for all the responses: - -```rs -async fn connecting_side(conn: Connection, request: &[u8]) -> Result<()> { - let mut send = conn.open_uni().await?; - send.write_all(request).await?; - send.finish()?; - - let recv_tasks = TaskTracker::new(); - // accept_uni will return `Ok(None)` once the connection is closed - loop { - match conn.accept_uni().await? { - Some(recv) => { - recv_tasks.spawn(handle_response(recv)); - } - None => break, - } - } - recv_tasks.wait().await; - conn.close(0u32.into(), b"Thank you!"); - - Ok(()) -} -``` - - -You might've noticed that this destroys the "association" between the two stream directions. This means we can't use tricks similar to what HTTP3 does that we described above to multiplex multiple request-responses interactions on the same connection. -This is unfortunate, but can be fixed by prefixing your requests and responses with a unique ID chosen per request. This ID then helps associate the responses to the requests that used the same ID. -Another thing that might or might not be important for your use case is knowing when unordered stream of responses is "done": -You can either introduce another message type that is interpreted as a finishing token, but there's another elegant way of solving this. Instead of only opening a uni-directional stream for the request, you open a bi-directional one. The response stream will only be used to indicate the final response stream ID. It then acts as a sort of "control stream" to provide auxiliary information about the request for the connecting endpoint. - - -## Time-sensitive Real-time interaction - -We often see users reaching for the QUIC datagram extension when implementing real-time protocols. Doing this is in most cases misguided. -QUIC datagram sending still interacts with QUIC's congestion controller and thus are also acknowledged. -Implementing traditional protocols on top of QUIC datagrams might thus not perform the way they were designed to. -Instead, it's often better to use lots of streams that are then stopped, reset or prioritized. - -A real-world example is the [media over QUIC protocol](https://doc.moq.dev/) (MoQ in short): MoQ is used -to transfer live video frames. It uses one QUIC stream for each frame (QUIC -streams are cheap to create)! - -The receiver then stops streams that are "too old" to be delivered, e.g. because it's a live video stream and newer frames were already fully received. -Similarly, the sending side will also reset older streams for the application level to indicate to the QUIC stack it doesn't need to keep re-trying the transmission of an outdated live video frame. (MoQ will actually also use stream prioritization to make sure the newest video frames get scheduled to be sent first.) - -## Closing Connections - -Gracefully closing connections can be tricky to get right when first working with the QUIC API. -If you don't close connections gracefully, you'll see the connecting timing out on one endpoint, usually after 30s, even though another endpoint finishes promptly without errors. -This happens when the endpoint that finishes doesn't notify the other endpoint about having finished operations. -There's mainly two reasons this happens: -1. The protocol doesn't call `Connection::close` at the right moment. -2. The endpoint that closes the connection is immediately dropped afterwards without waiting for `Endpoint::close`. -To make sure that you're not hitting (2), simply always make sure to wait for `Endpoint::close` to resolve, on both `Endpoint`s, if you can afford it. -Getting (1) right is harder. We might accidentally close connections too early, because we accidentally drop the `Connection` (which implicitly calls close). Instead, we should always keep around the connection and either wait for `Connection::closed` to resolve or call `Connection::close` ourselves at the right moment. When that is depends on what kind of protocol you're implementing: -### After a single Interaction - -Protocols that implement a single interaction want to keep their connection alive for only the time of this interaction. -In this case, the endpoint that received application data last will be the endpoint that calls `Connection::close` at that point in time. -Conversely, the other endpoint should wait for `Connection::closed` to resolve before ending its operations. -An example of this can be seen in the [Request and Response](#request-and-response) section above: The connecting side closes the connection once it received the response and the accepting side waits for the connection to be closed after having sent off the response. - -### During continuous Interaction - -Sometimes we want to keep open connections as long as the user is actively working with the application, so we don't needlessly run handshakes or try to hole-punch repeatedly. -In these cases, the protocol flow doesn't indicate which endpoint of the connection will be the one that closes the connection. -Instead, clients should concurrently monitor `Connection::closed` while they're running the protocol: - -```rs -async fn handle_connection(conn: Connection) -> Result<()> { - futures_lite::future::race( - run_protocol(conn.clone()), - async move { - conn.closed().await; - anyhow::Ok(()) - }, - ).await?; - Ok(()) -} - -async fn run_protocol(conn: Connection) -> Result<()> { - // run normal protocol flow - // once we realize we want to abort the connection flow - conn.close(0u32.into(), b"ah sorry, have to go!"); - - Ok(()) -} -``` - -And again, after `handle_connection` we need to make sure to wait for `Endpoint::close` to resolve. - -## Aborting Streams - -Sometimes you need to abandon a stream before it completes - either because the data has become stale, or because you've decided you no longer need it. QUIC provides mechanisms for both the sender and receiver to abort streams gracefully. - -### When to abort streams - -A real-world example comes from Media over QUIC (MoQ), which streams live video frames. Consider this scenario: - -- Each video frame is sent on its own uni-directional stream -- Frames arrive out of order due to network conditions -- By the time an old frame finishes transmitting, newer frames have already been received -- Continuing to receive the old frame wastes bandwidth and processing time - -### How to abort streams - -**From the sender side:** - -Use `SendStream::reset(error_code)` to immediately stop sending data and discard any buffered bytes. This tells QUIC to stop retrying lost packets for this stream. - - -**From the receiver side:** - -Use `RecvStream::stop(error_code)` to tell the sender you're no longer interested in the data. This allows the sender's QUIC stack to stop retransmitting lost packets. - - -### Key insights - -1. **Stream IDs indicate order**: QUIC stream IDs are monotonically increasing. You can compare stream IDs to determine which streams are newer without relying on application-level sequencing. - -2. **Both sides can abort**: Either the sender (via `reset`) or receiver (via `stop`) can abort a stream. Whichever side detects the data is no longer needed first should initiate the abort. - -3. **QUIC stops retransmissions**: When a stream is reset or stopped, QUIC immediately stops trying to recover lost packets for that stream, saving bandwidth and processing time. - -4. **Streams are cheap**: Opening a new stream is very fast (no round-trips required), so it's perfectly fine to open one stream per video frame, message, or other small unit of data. - -This pattern of using many short-lived streams that can be individually aborted is one of QUIC's most powerful features for real-time applications. It gives you fine-grained control over what data is worth transmitting, without the head-of-line blocking issues that would occur with a single TCP connection. - -## QUIC 0-RTT features - -### Server-side 0.5-RTT - -QUIC connections always take 1 full round-trip time (RTT) to establish - the client sends a hello, the server responds with its hello and certificate, and only then can application data flow. However, the server can actually start sending application data **before** the client finishes the handshake, achieving what's called "0.5-RTT" latency. - -This works because after the server sends its hello, it doesn't need to wait for the client's final handshake message before it can start processing the client's request and sending a response. The server knows the encryption keys at this point and can immediately begin sending data back. - -**How to use it:** - -On the server side, this happens automatically - you don't need to do anything special. As soon as you `accept_bi()` or `accept_uni()` a stream, you can start writing to it immediately, even if the handshake hasn't fully completed on the client side yet. - -```rs -async fn accepting_endpoint(conn: Connection) -> Result<()> { - let (mut send, mut recv) = conn.accept_bi().await?.ok_or_else(|| anyhow!("connection closed"))?; - let request = recv.read_to_end(MAX_REQUEST_SIZE).await?; - - // We can start sending the response immediately without waiting - // for the client to finish the handshake - let response = compute_response(&request); - send.write_all(&response).await?; - send.finish()?; - - conn.closed().await; - Ok(()) -} -``` - -**Important gotcha: Request replay attacks** - -Because the server starts processing before the handshake completes, there's a security consideration: the client's initial request data could potentially be replayed by an attacker who intercepts the handshake packets. This means: - -- **The server should treat 0.5-RTT requests as potentially non-idempotent** -- Avoid performing actions with side effects (like making payments, deleting data, etc.) based solely on 0.5-RTT data -- If your protocol requires idempotency guarantees, wait for the handshake to complete before processing sensitive operations - -For read-only operations or idempotent requests, 0.5-RTT is perfectly safe and provides a nice latency improvement.