From f6b27b1d2ce31f05a62c208292b1bc86c70b0788 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Mar 2026 14:18:50 -0800 Subject: [PATCH 01/25] Serve the init segment instead of pieces of it. (#1062) --- doc/spec/draft-ietf-moq-cmsf-00.txt | 504 +++++++ doc/spec/draft-ietf-moq-msf-00.txt | 1904 ++++++++++++++++++++++++++ js/hang/src/catalog/container.ts | 11 +- js/hang/src/container/cmaf/encode.ts | 21 +- js/watch/src/audio/decoder.ts | 4 +- js/watch/src/audio/mse.ts | 9 +- js/watch/src/video/decoder.ts | 4 +- js/watch/src/video/mse.ts | 9 +- rs/hang/src/catalog/container.rs | 11 +- rs/moq-mux/src/import/fmp4.rs | 61 +- rs/moq-mux/src/msf.rs | 28 +- 11 files changed, 2503 insertions(+), 63 deletions(-) create mode 100644 doc/spec/draft-ietf-moq-cmsf-00.txt create mode 100644 doc/spec/draft-ietf-moq-msf-00.txt diff --git a/doc/spec/draft-ietf-moq-cmsf-00.txt b/doc/spec/draft-ietf-moq-cmsf-00.txt new file mode 100644 index 000000000..a90ff58ca --- /dev/null +++ b/doc/spec/draft-ietf-moq-cmsf-00.txt @@ -0,0 +1,504 @@ + + + + +Media Over QUIC W. Law +Internet-Draft Akamai +Intended status: Informational 1 December 2025 +Expires: 4 June 2026 + + + CMSF- a CMAF compliant implementation of MOQT Streaming Format + draft-ietf-moq-cmsf-00 + +Abstract + + This document updates [MSF] by defining a new optional feature for + the streaming format. It specifies the syntax and semantics for + adding CMAF-packaged media [CMAF] to MSF. + +About This Document + + This note is to be removed before publishing as an RFC. + + The latest revision of this draft can be found at https://moq- + wg.github.io/cmsf/draft-wilaw-moq-cmsf.html. Status information for + this document may be found at https://datatracker.ietf.org/doc/draft- + ietf-moq-cmsf/. + + Discussion of this document takes place on the Media Over QUIC + Working Group mailing list (mailto:moq@ietf.org), which is archived + at https://mailarchive.ietf.org/arch/browse/moq/. Subscribe at + https://www.ietf.org/mailman/listinfo/moq/. + + Source for this draft and an issue tracker can be found at + https://github.com/moq-wg/cmsf. + +Status of This Memo + + This Internet-Draft is submitted in full conformance with the + provisions of BCP 78 and BCP 79. + + Internet-Drafts are working documents of the Internet Engineering + Task Force (IETF). Note that other groups may also distribute + working documents as Internet-Drafts. The list of current Internet- + Drafts is at https://datatracker.ietf.org/drafts/current/. + + Internet-Drafts are draft documents valid for a maximum of six months + and may be updated, replaced, or obsoleted by other documents at any + time. It is inappropriate to use Internet-Drafts as reference + material or to cite them other than as "work in progress." + + This Internet-Draft will expire on 4 June 2026. + + + +Law Expires 4 June 2026 [Page 1] + +Internet-Draft CMSF- a CMAF compliant implementation of December 2025 + + +Copyright Notice + + Copyright (c) 2025 IETF Trust and the persons identified as the + document authors. All rights reserved. + + This document is subject to BCP 78 and the IETF Trust's Legal + Provisions Relating to IETF Documents (https://trustee.ietf.org/ + license-info) in effect on the date of publication of this document. + Please review these documents carefully, as they describe your rights + and restrictions with respect to this document. Code Components + extracted from this document must include Revised BSD License text as + described in Section 4.e of the Trust Legal Provisions and are + provided without warranty as described in the Revised BSD License. + +Table of Contents + + 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . 2 + 2. MSF Extension . . . . . . . . . . . . . . . . . . . . . . . . 3 + 3. CMAF Packaging . . . . . . . . . . . . . . . . . . . . . . . 3 + 3.1. Initialization headers . . . . . . . . . . . . . . . . . 3 + 3.2. Switching sets and tracks . . . . . . . . . . . . . . . . 3 + 3.3. Object Packaging . . . . . . . . . . . . . . . . . . . . 3 + 3.4. Group Packaging . . . . . . . . . . . . . . . . . . . . . 4 + 3.5. Catalog description . . . . . . . . . . . . . . . . . . . 4 + 3.5.1. CMAF packaging type . . . . . . . . . . . . . . . . . 4 + 3.5.2. Max SAP starting types . . . . . . . . . . . . . . . 4 + 3.6. Event Timelines . . . . . . . . . . . . . . . . . . . . . 5 + 3.6.1. SAP Type timeline . . . . . . . . . . . . . . . . . . 5 + 3.6.2. SAP-type timeline track example . . . . . . . . . . . 6 + 4. Catalog Examples . . . . . . . . . . . . . . . . . . . . . . 6 + 4.1. Simulcast video tracks - 3 alternate video qualities along + with audio . . . . . . . . . . . . . . . . . . . . . . . 7 + 5. Conventions and Definitions . . . . . . . . . . . . . . . . . 8 + 6. Security Considerations . . . . . . . . . . . . . . . . . . . 8 + 7. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 8 + 8. Normative References . . . . . . . . . . . . . . . . . . . . 8 + Acknowledgments . . . . . . . . . . . . . . . . . . . . . . . . . 9 + Author's Address . . . . . . . . . . . . . . . . . . . . . . . . 9 + +1. Introduction + + CMAF compliant MOQT Streaming Format (CMSF) is a media format + designed to deliver CMAF [CMAF] and LOC [LOC] compliant media content + over MOQ Transport (MOQT) [MoQTransport]. CMSF extends MSF and + retains all the scope, capabilities and features of MSF including the + catalog format, timeline, ABR switching and LOC support. MSF is + targeted at real-time and interactive levels of live latency, as well + as VOD content. + + + +Law Expires 4 June 2026 [Page 2] + +Internet-Draft CMSF- a CMAF compliant implementation of December 2025 + + + This document describes version 1 of the CMSF streaming format. + +2. MSF Extension + + All of the specifications, requirements, and terminology defined in + [MSF] apply to implementations of this extension unless explicitly + noted otherwise in this document. + +3. CMAF Packaging + +3.1. Initialization headers + + A CMAF header is a sequence of CMAF constrained ISO BMFF boxes that + do not reference any media samples, but are associated with a CMAF + track and are necessary for initializing the decoding of the + subsequent CMAF fragments. + + The header for a given MOQT Track MUST be packaged by encoding the + header using [BASE64] and then inserting that payload as the value of + the Initialization data "initData" field in the catalog entry for + that Track. + +3.2. Switching sets and tracks + + This specification defines a direct mapping between CMAF Tracks ( + [CMAF] Sect 3.2.1) and MOQT tracks ([MoQTransport] Sect 2.3). + + CMAF switching sets are a set of one or more CMAF tracks (3.2.1), + where each track is an alternative encoding of the same source + content and are constrained to enable seamless track switching + (3.3.9). + + Each CMAF track in a switching set MUST be transmitted as a separate + MOQT Track. The catalog entry for each of these tracks in the + switching set MUST carry a Alternate group (altGroup) key with a + common value. + + The MOQT Group numbers within these switching set tracks MUST be + media time-aligned. Mandating the track being media time-aligned + requires that the presentation time of the first media sample + contained within the first MOQT Object of each MOQT Group is + identical. + +3.3. Object Packaging + + The payload of each Object is subject to the following requirements: + + + + + +Law Expires 4 June 2026 [Page 3] + +Internet-Draft CMSF- a CMAF compliant implementation of December 2025 + + + * MUST contain at least one Movie Fragment Box (moof) followed by a + Media Data Box (mdat). This is equivalent to requiring that each + Object hold at least one CMAF Chunk. The Media Fragment Box + (moof) MUST contain a Movie Fragment Header Box (mfhd) and Track + Box (trak) with a Track ID (track_ID) matching a Track Box in the + initialization fragment. + + * MAY contain multiple successive CMAF Chunks. + + * MUST contain a single [ISOBMFF] track. + + * MUST contain media content encoded in decode order. + +3.4. Group Packaging + + Each MOQT Group + + * MUST begin with an Object containing a stream access point (SAP + type 1 or 2). + + * MUST contain one or more contiguous Groups of Pictures (GOPs). + + * The Group boundary MUST align with a CMAF Fragment boundary. CMAF + Fragments and CMAF Chunks MUST not span Groups. + +3.5. Catalog description + +3.5.1. CMAF packaging type + + This specification extends the allowed packaging values defined in + [MSF] to include one new entry, as defined in Table 1 below: + + +======+=======+===========+ + | Name | Value | Reference | + +======+=======+===========+ + | CMAF | cmaf | This RFC | + +------+-------+-----------+ + + Table 1 + + Every Track entry in a CMSF catalog carrying CMAF-packaged media data + MUST declare a "packaging" type value of "cmaf". + +3.5.2. Max SAP starting types + + This specification adds two track-level catalog fields, as defined in + Table 2 below: + + + + +Law Expires 4 June 2026 [Page 4] + +Internet-Draft CMSF- a CMAF compliant implementation of December 2025 + + + +=============================+=======================+============+ + | Field | Name | Definition | + +=============================+=======================+============+ + | Max Group SAP starting type | maxGrpSapStartingType | Section | + | | | 3.5.2.1 | + +-----------------------------+-----------------------+------------+ + | Max Object SAP starting | maxObjSapStartingType | Section | + | type | | 3.5.2.2 | + +-----------------------------+-----------------------+------------+ + + Table 2 + +3.5.2.1. Max Group SAP starting type + + Location: T Required: Optional JSON Type: Number + + A number indicating the maximum SAP type the MOQT Groups in the track + start with. + +3.5.2.2. Max Object SAP starting type + + Location: T Required: Optional JSON Type: Number + + A number indicating the maximum SAP type the MOQT Objects in the + track start with. + +3.6. Event Timelines + +3.6.1. SAP Type timeline + + CMSF defines a special instance of an Event Timeline track, termed + the SAP Type timeline track. Its purpose is to convey information + about the distribution of Stream Access Point types and their + associated Earlist Presentation Times. + + In the catalog, the SAP-type timeline track MUST include a + 'packaging' value of 'eventtimeline" and MUST include an 'eventType' + value of 'org.ietf.moq.cmsf.sap'. + + In the SAP Type timeline JSON payload: + + * The index reference MUST be 'l' for Location + + * The data field is a JSON Array containing two integers. The first + integer defines SAP type with an allowed value of 0,1,2 or 3. The + value 0 indicates that the Object does not start with an ISOBMFF + stream access point. The value equal to 1, 2, or 3 indicates that + the Object begins with a stream access point of SAP type 1, 2, or + + + +Law Expires 4 June 2026 [Page 5] + +Internet-Draft CMSF- a CMAF compliant implementation of December 2025 + + + 3, respectively. When the Object is the first Object in the + Group, the value MUST be equal to 1 or 2. The second integer + defines the earliest media presentation timestamp, rounded to the + nearest millisecond, of all media samples in the Object defined by + the Location of that record. + +3.6.2. SAP-type timeline track example + + This shows an example of 30-fps HEVC-encoded content, in which each + 4s Group beings with SAP-type 2 (i.e., the first picture in the Group + is an IDR picture, while there may be one or more pictures in the + Group following the IDR picture in decoding order but preceding it in + output order). After 2 seconds in each Group, there is a SAP-type 3, + i.e., a CRA picture, which is associated with one or more Random + Access Skipped Leading (RASL) pictures. A small buffer of frames (10 + frames at 30 fps) is skipped/discarded (RASL pictures) when the + streaming session starts from the SAP-type 3 location. In this + example, the EPT is the presentation time of the first picture after + the RASL pictures in decoding order; all pictures after the RASL + pictures can be fully correctly decoded and are thus presentable when + the streaming session starts from the SAP-type 3 location. Note that + if the streaming session starts from the start of the Group, then + these RASL pictures can be fully correctly decoded and are thus + presentable. + + [ + { + "l": [0,0], + "data": [2,0] + }, + { + "l": [0,60], + "data": [3,2100] + }, + { + "l": [1,0], + "data": [2,4000] + }, + { + "l": [1,60], + "data": [3,6100] + } + ] + +4. Catalog Examples + + The following section provides non-normative JSON examples of various + catalogs compliant with this draft. + + + +Law Expires 4 June 2026 [Page 6] + +Internet-Draft CMSF- a CMAF compliant implementation of December 2025 + + +4.1. Simulcast video tracks - 3 alternate video qualities along with + audio + + This example shows catalog for a media producer capable of sending 3 + time-aligned video tracks for high definition, low definition and + medium definition video qualities, along with an audio track. + + { + "version": 1, + "generatedAt": 1746104606044, + "tracks":[ + { + "name": "hd", + "renderGroup": 1, + "packaging": "cmaf", + "isLive": true, + "initData": "AAAAIGZ0eXBpc281AAA...AAAAAAAAAAAAA", + "role": "video", + "codec":"avc1.640028", + "width":1920, + "height":1080, + "bitrate":5000000, + "framerate":30, + "altGroup":1 + }, + { + "name": "md", + "renderGroup": 1, + "packaging": "cmaf", + "isLive": true, + "initData": "AAAAHGZ0eXBpc281AAA...AAAAAAAAAAAAAA", + "role": "video", + "codec":"avc1.64001e", + "width":720, + "height":640, + "bitrate":3000000, + "framerate":30, + "altGroup":1 + }, + { + "name": "sd", + "renderGroup": 1, + "packaging": "cmaf", + "isLive": true, + "initData": "AAAAHGZ0eXBpc281AAA...AAAAAAAAAAAAAA", + "role": "video", + "codec":"avc1.64000d", + "width":192, + + + +Law Expires 4 June 2026 [Page 7] + +Internet-Draft CMSF- a CMAF compliant implementation of December 2025 + + + "height":144, + "bitrate":500000, + "framerate":30, + "altGroup":1 + }, + { + "name": "audio", + "renderGroup": 1, + "packaging": "cmaf", + "isLive": true, + "initData": "AAAAHGZ0eXBpc281AAA...AAAAAAAAAAAAAA", + "role": "audio", + "codec":"mp4a.40.5", + "samplerate":48000, + "channelConfig":"2", + "bitrate":67071 + } + ] + } + +5. Conventions and Definitions + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "NOT RECOMMENDED", "MAY", and + "OPTIONAL" in this document are to be interpreted as described in + BCP 14 [RFC2119] [RFC8174] when, and only when, they appear in all + capitals, as shown here. + +6. Security Considerations + + TODO Security + +7. IANA Considerations + + This document has no IANA actions. + +8. Normative References + + [BASE64] Josefsson, S., "The Base16, Base32, and Base64 Data + Encodings", RFC 4648, DOI 10.17487/RFC4648, October 2006, + . + + [CMAF] Standardization, I. O. for., "Information technology — + Multimedia application format (MPEG-A) — Part 19: Common + media application format (CMAF) for segmented media", + October 2021. + + + + + +Law Expires 4 June 2026 [Page 8] + +Internet-Draft CMSF- a CMAF compliant implementation of December 2025 + + + [MoQTransport] + Curley, L., Pugin, K., Nandakumar, S., Vasiliev, V., and + I. Swett, "Media over QUIC Transport", Work in Progress, + Internet-Draft, draft-ietf-moq-transport-10, 3 March 2025, + . + + [MSF] Law, W., Curley, L., Vasiliev, V., Nandakumar, S., and K. + Pugin, "WARP Streaming Format", Work in Progress, + Internet-Draft, draft-ietf-moq-warp-01, 22 July 2025, + . + + [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, + DOI 10.17487/RFC2119, March 1997, + . + + [RFC8174] Leiba, B., "Ambiguity of Uppercase vs Lowercase in RFC + 2119 Key Words", BCP 14, RFC 8174, DOI 10.17487/RFC8174, + May 2017, . + +Acknowledgments + + TODO acknowledge. + +Author's Address + + Will Law + Akamai + Email: wilaw@akamai.com + + + + + + + + + + + + + + + + + + + + +Law Expires 4 June 2026 [Page 9] diff --git a/doc/spec/draft-ietf-moq-msf-00.txt b/doc/spec/draft-ietf-moq-msf-00.txt new file mode 100644 index 000000000..aca1bf016 --- /dev/null +++ b/doc/spec/draft-ietf-moq-msf-00.txt @@ -0,0 +1,1904 @@ + + + + +Media Over QUIC W. Law +Internet-Draft Akamai +Intended status: Informational 19 January 2026 +Expires: 23 July 2026 + + + MOQT Streaming Format + draft-ietf-moq-msf-00 + +Abstract + + This document specifies the MOQT Streaming Format, designed to + operate on Media Over QUIC Transport. + +About This Document + + This note is to be removed before publishing as an RFC. + + The latest revision of this draft can be found at https://moq- + wg.github.io/msf/draft-ietf-moq-msf.html. Status information for + this document may be found at https://datatracker.ietf.org/doc/draft- + ietf-moq-msf/. + + Discussion of this document takes place on the Media Over QUIC + Working Group mailing list (mailto:moq@ietf.org), which is archived + at https://mailarchive.ietf.org/arch/browse/moq/. Subscribe at + https://www.ietf.org/mailman/listinfo/moq/. + + Source for this draft and an issue tracker can be found at + https://github.com/moq-wg/msf. + +Status of This Memo + + This Internet-Draft is submitted in full conformance with the + provisions of BCP 78 and BCP 79. + + Internet-Drafts are working documents of the Internet Engineering + Task Force (IETF). Note that other groups may also distribute + working documents as Internet-Drafts. The list of current Internet- + Drafts is at https://datatracker.ietf.org/drafts/current/. + + Internet-Drafts are draft documents valid for a maximum of six months + and may be updated, replaced, or obsoleted by other documents at any + time. It is inappropriate to use Internet-Drafts as reference + material or to cite them other than as "work in progress." + + This Internet-Draft will expire on 23 July 2026. + + + + +Law Expires 23 July 2026 [Page 1] + +Internet-Draft MOQT Streaming Format January 2026 + + +Copyright Notice + + Copyright (c) 2026 IETF Trust and the persons identified as the + document authors. All rights reserved. + + This document is subject to BCP 78 and the IETF Trust's Legal + Provisions Relating to IETF Documents (https://trustee.ietf.org/ + license-info) in effect on the date of publication of this document. + Please review these documents carefully, as they describe your rights + and restrictions with respect to this document. Code Components + extracted from this document must include Revised BSD License text as + described in Section 4.e of the Trust Legal Provisions and are + provided without warranty as described in the Revised BSD License. + +Table of Contents + + 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . 4 + 2. Conventions and Definitions . . . . . . . . . . . . . . . . . 4 + 3. Scope . . . . . . . . . . . . . . . . . . . . . . . . . . . . 4 + 4. Media packaging . . . . . . . . . . . . . . . . . . . . . . . 6 + 4.1. LOC packaging . . . . . . . . . . . . . . . . . . . . . . 6 + 4.2. Time-alignment . . . . . . . . . . . . . . . . . . . . . 6 + 4.3. Content protection and encryption . . . . . . . . . . . . 6 + 5. Catalog . . . . . . . . . . . . . . . . . . . . . . . . . . . 6 + 5.1. Catalog Fields . . . . . . . . . . . . . . . . . . . . . 7 + 5.1.1. MSF version . . . . . . . . . . . . . . . . . . . . . 9 + 5.1.2. Delta update . . . . . . . . . . . . . . . . . . . . 9 + 5.1.3. Add tracks . . . . . . . . . . . . . . . . . . . . . 9 + 5.1.4. Remove tracks . . . . . . . . . . . . . . . . . . . . 9 + 5.1.5. Clone tracks . . . . . . . . . . . . . . . . . . . . 10 + 5.1.6. Generated at . . . . . . . . . . . . . . . . . . . . 10 + 5.1.7. Is Complete . . . . . . . . . . . . . . . . . . . . . 10 + 5.1.8. Tracks . . . . . . . . . . . . . . . . . . . . . . . 10 + 5.1.9. Tracks object . . . . . . . . . . . . . . . . . . . . 10 + 5.1.10. Track namespace . . . . . . . . . . . . . . . . . . . 11 + 5.1.11. Track name . . . . . . . . . . . . . . . . . . . . . 11 + 5.1.12. Packaging . . . . . . . . . . . . . . . . . . . . . . 11 + 5.1.13. Event timeline type . . . . . . . . . . . . . . . . . 11 + 5.1.14. Track role . . . . . . . . . . . . . . . . . . . . . 12 + 5.1.15. Is Live . . . . . . . . . . . . . . . . . . . . . . . 13 + 5.1.16. Target latency . . . . . . . . . . . . . . . . . . . 13 + 5.1.17. Track label . . . . . . . . . . . . . . . . . . . . . 13 + 5.1.18. Render group . . . . . . . . . . . . . . . . . . . . 13 + 5.1.19. Alternate group . . . . . . . . . . . . . . . . . . . 14 + 5.1.20. Initialization data . . . . . . . . . . . . . . . . . 14 + 5.1.21. Dependencies . . . . . . . . . . . . . . . . . . . . 14 + 5.1.22. Temporal ID . . . . . . . . . . . . . . . . . . . . . 14 + 5.1.23. Spatial ID . . . . . . . . . . . . . . . . . . . . . 14 + + + +Law Expires 23 July 2026 [Page 2] + +Internet-Draft MOQT Streaming Format January 2026 + + + 5.1.24. Codec . . . . . . . . . . . . . . . . . . . . . . . . 15 + 5.1.25. Mimetype . . . . . . . . . . . . . . . . . . . . . . 15 + 5.1.26. Framerate . . . . . . . . . . . . . . . . . . . . . . 15 + 5.1.27. Timescale . . . . . . . . . . . . . . . . . . . . . . 15 + 5.1.28. Bitrate . . . . . . . . . . . . . . . . . . . . . . . 15 + 5.1.29. Width . . . . . . . . . . . . . . . . . . . . . . . . 15 + 5.1.30. Height . . . . . . . . . . . . . . . . . . . . . . . 15 + 5.1.31. Audio sample rate . . . . . . . . . . . . . . . . . . 16 + 5.1.32. Channel configuration . . . . . . . . . . . . . . . . 16 + 5.1.33. Display width . . . . . . . . . . . . . . . . . . . . 16 + 5.1.34. Display height . . . . . . . . . . . . . . . . . . . 16 + 5.1.35. Language . . . . . . . . . . . . . . . . . . . . . . 16 + 5.1.36. Parent name . . . . . . . . . . . . . . . . . . . . . 16 + 5.1.37. Track duration . . . . . . . . . . . . . . . . . . . 17 + 5.2. Delta updates . . . . . . . . . . . . . . . . . . . . . . 17 + 5.3. Catalog Examples . . . . . . . . . . . . . . . . . . . . 18 + 5.3.1. Time-aligned Audio/Video Tracks with single + quality . . . . . . . . . . . . . . . . . . . . . . . 18 + 5.3.2. Simulcast video tracks - 3 alternate qualities along + with audio . . . . . . . . . . . . . . . . . . . . . 19 + 5.3.3. SVC video tracks with 2 spatial and 2 temporal + qualities . . . . . . . . . . . . . . . . . . . . . . 21 + 5.3.4. Delta update - adding two tracks . . . . . . . . . . 23 + 5.3.5. Delta update removing tracks . . . . . . . . . . . . 24 + 5.3.6. Time-aligned Audio/Video Tracks with custom field + values . . . . . . . . . . . . . . . . . . . . . . . 24 + 5.3.7. Time-aligned VOD Audio/Video Tracks . . . . . . . . . 25 + 5.3.8. Media timeline and Event timeline . . . . . . . . . . 26 + 5.3.9. Terminating a live broadcast . . . . . . . . . . . . 27 + 6. Media transmission . . . . . . . . . . . . . . . . . . . . . 28 + 6.1. Group numbering . . . . . . . . . . . . . . . . . . . . . 28 + 7. Media Timeline track . . . . . . . . . . . . . . . . . . . . 28 + 7.1. Media Timeline track payload . . . . . . . . . . . . . . 28 + 7.2. Media Timeline Catalog requirements . . . . . . . . . . . 29 + 7.3. Media Timeline track updating . . . . . . . . . . . . . . 29 + 8. Event Timeline track . . . . . . . . . . . . . . . . . . . . 29 + 8.1. Event Timeline data format . . . . . . . . . . . . . . . 30 + 8.2. Event Timeline Catalog requirements . . . . . . . . . . . 30 + 8.3. Event Timeline track updating . . . . . . . . . . . . . . 31 + 8.4. Event timeline track examples . . . . . . . . . . . . . . 31 + 8.4.1. Event timeline track with wallclock time indexing . . 31 + 8.4.2. Event timeline track with MOQT Location indexing . . 31 + 9. Workflow . . . . . . . . . . . . . . . . . . . . . . . . . . 32 + 9.1. Initiating a broadcast . . . . . . . . . . . . . . . . . 32 + 9.2. Ending a live broadcast . . . . . . . . . . . . . . . . . 32 + 10. Security Considerations . . . . . . . . . . . . . . . . . . . 32 + 11. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 32 + 12. Normative References . . . . . . . . . . . . . . . . . . . . 33 + + + +Law Expires 23 July 2026 [Page 3] + +Internet-Draft MOQT Streaming Format January 2026 + + + Acknowledgments . . . . . . . . . . . . . . . . . . . . . . . . . 34 + Contributors . . . . . . . . . . . . . . . . . . . . . . . . . . 34 + Author's Address . . . . . . . . . . . . . . . . . . . . . . . . 34 + +1. Introduction + + MOQT Streaming Format (MSF) is a media format designed to deliver LOC + [LOC] compliant media content over Media Over QUIC Transport (MOQT) + [MoQTransport]. MSF works by fragmenting the bitstream into objects + that can be independently transmitted. MSF leverages a catalog + format to describe the output of the original publisher. MSF + specifies how content should be packaged and signaled, defines how + the catalog communicates the content, specifies prioritization + strategies for real-time and workflows for beginning and terminating + broadcasts. MSF also details how end-subscribers may perform + adaptive bitrate switching. MSF is targeted at real-time and + interactive levels of live latency, as well as VOD content. + + This document describes version 1 of the streaming format. + +2. Conventions and Definitions + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "NOT RECOMMENDED", "MAY", and + "OPTIONAL" in this document are to be interpreted as described in + BCP 14 [RFC2119] [RFC8174] when, and only when, they appear in all + capitals, as shown here. + + This document uses the conventions detailed in Section 1.3 of + [RFC9000] when describing the binary encoding. + +3. Scope + + The purpose of MSF is to provide an interoperable media streaming + format operating over [MoQTransport]. Interoperability implies that: + + * An original publisher can package incoming media content into + tracks, prepare a catalog and announce the availability of the + content to an MOQT relay. Media content refers to audio and video + data, as well as ancillary data such as captions, subtitles, + accessibility and other timed-text data. + + * An MOQT relay can process the announcement as well as cache and + propagate the tracks, both to other relays or to the final + subscriber. + + * A final subscriber can parse the catalog, request tracks, decode + and render the received media data. + + + +Law Expires 23 July 2026 [Page 4] + +Internet-Draft MOQT Streaming Format January 2026 + + + MSF is intended to provide a format for delivering commercial media + content. To that end, the following features are within scope: + + * Video codecs - all codecs supported by [LOC] + + * Audio codecs - all audio codecs supported by [LOC] + + * Catalog track - describes the availability and characteristics of + content produced by the original publisher. + + * Timeline track - describes the relationship between MOQT Group and + Object IDs to media time. + + * Token-based authorization and access control + + * Captions + Subtitles - support for [WEBVTT] and [IMSC1] + transmission + + * Latency support across multiple regimes (thresholds are + informative only and describe the delay between the original + publisher placing the content on the wire and the final subscriber + rendering it) + + * Real-time - less than 500ms + + * Interactive - between 500ms and 2500ms + + * Standard - above 2500ms + + * VOD latency - content that was previously produced, is no longer + live and is available indefinitely. + + * Content encryption + + * ABR between time-synced tracks - subscribers may switch between + tracks at different quality levels in order to maximize visual or + audio quality under conditions of throughput variability. + + * Capable of delivering interstitial advertising. + + * Logs and analytics management - support for the reporting of + client-side QoE and relay delivery actions. + + Initial versions of MSF will prioritize basic features necessary to + exercise interoperability across delivery systems. Later versions + will add commercially necessary features. + + + + + +Law Expires 23 July 2026 [Page 5] + +Internet-Draft MOQT Streaming Format January 2026 + + +4. Media packaging + + MSF delivers LOC [LOC] packaged media bitstreams. + +4.1. LOC packaging + + This specification references Low Overhead Container (LOC) [LOC] to + define how audio and video content is packaged. With this packaging + mode, each EncodedAudioChunk or EncodedVideoChunk sample is placed in + a separate MOQT Object. Samples that belong to the same Group of + Pictures (GOP) MUST be placed within the same MOQT Group. + + When LOC packaging is used for a track, the catalog packaging + attribute (Section 5.1.12) MUST be present and it MUST be populated + with a value of "loc". + +4.2. Time-alignment + + MSF Tracks MAY be time-aligned. Those that are, are subject to the + following requirements: + + * Tracks advertised in the catalog as belonging to a common render + group MUST be time-aligned. + + * The render duration of the first media object of each equally + numbered MOQT Group, after decoding, MUST have overlapping + presentation time. + + A consequence of this restriction is that an MSF receiver SHOULD be + able to cleanly switch between time-aligned media tracks at group + boundaries. + +4.3. Content protection and encryption + + ToDo - content protection for LOC-packaged content. + +5. Catalog + + A Catalog is an MOQT Track that provides information about the other + tracks being produced by a MSF publisher. A Catalog is used by MSF + publishers for advertising their output and for subscribers in + consuming that output. The payload of the Catalog object is opaque + to Relays and can be end-to-end encrypted. The Catalog provides the + names and namespaces of the tracks being produced, along with the + relationship between tracks, properties of the tracks that consumers + may use for selection and any relevant initialization data. + + The catalog track MUST have a case-sensitive Track Name of "catalog". + + + +Law Expires 23 July 2026 [Page 6] + +Internet-Draft MOQT Streaming Format January 2026 + + + A catalog object MAY be independent of other catalog objects or it + MAY represent a delta update of a prior catalog object. The first + catalog object published within a new group MUST be independent. A + catalog object SHOULD be published only when the availability of + tracks changes. + + Each catalog update MUST be mapped to an MOQT Object. + +5.1. Catalog Fields + + A catalog is a JSON [JSON] document, comprised of a series of + mandatory and optional fields. At a minimum, a catalog MUST provide + all mandatory fields. A producer MAY add additional fields to the + ones described in this draft. Custom field names MUST NOT collide + with field names described in this draft. The order of field names + within the JSON document is not important. + + A parser MUST ignore fields it does not understand. + + Table 1 provides an overview of all fields defined by this document. + + +=======================+===============+================+ + | Field | Name | Definition | + +=======================+===============+================+ + | MSF version | version | Section 5.1.1 | + +-----------------------+---------------+----------------+ + | Delta update | deltaUpdate | Section 5.1.2 | + +-----------------------+---------------+----------------+ + | Add tracks | addTracks | Section 5.1.3 | + +-----------------------+---------------+----------------+ + | Remove tracks | removeTracks | Section 5.1.4 | + +-----------------------+---------------+----------------+ + | Clone tracks | cloneTracks | Section 5.1.5 | + +-----------------------+---------------+----------------+ + | Generated at | generatedAt | Section 5.1.6 | + +-----------------------+---------------+----------------+ + | Is Complete | isComplete | Section 5.1.7 | + +-----------------------+---------------+----------------+ + | Tracks | tracks | Section 5.1.8 | + +-----------------------+---------------+----------------+ + | Track namespace | namespace | Section 5.1.10 | + +-----------------------+---------------+----------------+ + | Track name | name | Section 5.1.11 | + +-----------------------+---------------+----------------+ + | Packaging | packaging | Section 5.1.12 | + +-----------------------+---------------+----------------+ + | Event timeline type | eventType | Section 5.1.13 | + +-----------------------+---------------+----------------+ + + + +Law Expires 23 July 2026 [Page 7] + +Internet-Draft MOQT Streaming Format January 2026 + + + | Is Live | isLive | Section 5.1.15 | + +-----------------------+---------------+----------------+ + | Target latency | targetLatency | Section 5.1.16 | + +-----------------------+---------------+----------------+ + | Track role | role | Section 5.1.14 | + +-----------------------+---------------+----------------+ + | Track label | label | Section 5.1.17 | + +-----------------------+---------------+----------------+ + | Render group | renderGroup | Section 5.1.18 | + +-----------------------+---------------+----------------+ + | Alternate group | altGroup | Section 5.1.19 | + +-----------------------+---------------+----------------+ + | Initialization data | initData | Section 5.1.20 | + +-----------------------+---------------+----------------+ + | Dependencies | depends | Section 5.1.21 | + +-----------------------+---------------+----------------+ + | Temporal ID | temporalId | Section 5.1.22 | + +-----------------------+---------------+----------------+ + | Spatial ID | spatialId | Section 5.1.23 | + +-----------------------+---------------+----------------+ + | Codec | codec | Section 5.1.24 | + +-----------------------+---------------+----------------+ + | Mime type | mimeType | Section 5.1.25 | + +-----------------------+---------------+----------------+ + | Framerate | framerate | Section 5.1.26 | + +-----------------------+---------------+----------------+ + | Timescale | timescale | Section 5.1.27 | + +-----------------------+---------------+----------------+ + | Bitrate | bitrate | Section 5.1.28 | + +-----------------------+---------------+----------------+ + | Width | width | Section 5.1.29 | + +-----------------------+---------------+----------------+ + | Height | height | Section 5.1.30 | + +-----------------------+---------------+----------------+ + | Audio sample rate | samplerate | Section 5.1.31 | + +-----------------------+---------------+----------------+ + | Channel configuration | channelConfig | Section 5.1.32 | + +-----------------------+---------------+----------------+ + | Display width | displayWidth | Section 5.1.33 | + +-----------------------+---------------+----------------+ + | Display height | displayHeight | Section 5.1.34 | + +-----------------------+---------------+----------------+ + | Language | lang | Section 5.1.35 | + +-----------------------+---------------+----------------+ + | Parent name | parentName | Section 5.1.36 | + +-----------------------+---------------+----------------+ + | Track duration | trackDuration | Section 5.1.37 | + +-----------------------+---------------+----------------+ + + + +Law Expires 23 July 2026 [Page 8] + +Internet-Draft MOQT Streaming Format January 2026 + + + Table 1 + + Table 2 defines the allowed locations for these fields within the + document + + +==========+=================================+ + | Location | Allowed locations for the field | + +==========+=================================+ + | R | The Root of the JSON object | + +----------+---------------------------------+ + | T | Track object | + +----------+---------------------------------+ + + Table 2 + +5.1.1. MSF version + + Location: R Required: Yes JSON Type: Number + + Specifies the version of MSF referenced by this catalog. There is no + guarantee that future catalog versions are backwards compatible and + field definitions and interpretation may change between versions. A + subscriber MUST NOT attempt to parse a catalog version which it does + not understand. + +5.1.2. Delta update + + Location: R Required: Optional JSON Type: Boolean + + A Boolean that if true indicates that this catalog object represents + a delta (or partial) update. A delta update has a restricted set of + fields and special processing rules - see Section 5.2. This value + SHOULD NOT be added to a catalog if it is false. + +5.1.3. Add tracks + + Location: R Required: Optional JSON Type: Array + + Indicates a delta processing instruction to add new tracks. The + value of this field is an Array of track objects Section 5.1.9. + +5.1.4. Remove tracks + + Location: R Required: Optional JSON Type: Array + + + + + + + +Law Expires 23 July 2026 [Page 9] + +Internet-Draft MOQT Streaming Format January 2026 + + + Indicates a delta processing instruction to remove new tracks. The + value of this field is an Array of track objects Section 5.1.9. Each + track object MUST include a Track Name Section 5.1.11 field, MAY + include a Track Namespace Section 5.1.10 field and MUST NOT hold any + other fields. + +5.1.5. Clone tracks + + Location: R Required: Optional JSON Type: Array + + Indicates a delta processing instruction to clone new tracks from + previously declared tracks. The value of this field is an Array of + track objects Section 5.1.9. Each track object MUST include a Parent + Name Section 5.1.36 field. + +5.1.6. Generated at + + Location: R Required: Optional JSON Type: Number + + The wallclock time at which this catalog instance was generated, + expressed as the number of milliseconds that have elapsed since + January 1, 1970 (midnight UTC/GMT). This field SHOULD NOT be + included if the isLive field is false. + +5.1.7. Is Complete + + Location: R Required: Optional JSON Type: Boolean + + Issued once a previously live broadcast is complete. This is a + commitment that all tracks are complete, no new tracks will be added + and no new content will be published. This field MUST NOT be + included if it is FALSE. This field MUST NOT be removed from a + catalog once it has been added. + +5.1.8. Tracks + + Location: R Required: Yes JSON Type: Array + + An array of track objects Section 5.1.9. + +5.1.9. Tracks object + + A track object is JSON Object containing a collection of fields whose + location is specified 'T' in Table 2. + + + + + + + +Law Expires 23 July 2026 [Page 10] + +Internet-Draft MOQT Streaming Format January 2026 + + +5.1.10. Track namespace + + Location: T Required: Optional JSON Type: String + + The name space under which the track name is defined. See section + 2.3 of [MoQTransport]. The track namespace is optional. If it is + not declared within a track, then each track MUST inherit the + namespace of the catalog track. A namespace declared in a track + object overwrites any inherited name space. + +5.1.11. Track name + + Location: T Required: Yes JSON Type: String + + A string defining the name of the track. See section 2.3 of + [MoQTransport]. Within the catalog, track names MUST be unique per + namespace. + +5.1.12. Packaging + + Location: T Required: Yes JSON Type: String + + A string defining the type of payload encapsulation. Allowed values + are strings as defined in Table 3. + + Table 3: Allowed packaging values + + +================+===============+===============+ + | Name | Value | Reference | + +================+===============+===============+ + | LOC | loc | See RFC XXXX | + +----------------+---------------+---------------+ + | Media Timeline | mediatimeline | See Section 7 | + +----------------+---------------+---------------+ + | Event Timeline | eventtimeline | See Section 8 | + +----------------+---------------+---------------+ + + Table 3 + +5.1.13. Event timeline type + + Location: T Required: Optional JSON Type: String + + + + + + + + + +Law Expires 23 July 2026 [Page 11] + +Internet-Draft MOQT Streaming Format January 2026 + + + A String defining the type & structure of the data contained within + the data field of the Event timeline track. Types are defined by the + application provider and are not centrally registered. Implementers + are encouraged to use a unique naming scheme, such as Reverse Domain + Name Notation, to avoid naming collisions. This field is required if + the Section 5.1.12 value is "eventtimeline". This field MUST NOT be + used if the packaging value is not "eventtimeline". + +5.1.14. Track role + + Location: T Required: Optional JSON Type: String + + A string defining the role of content carried by the track. + Specified roles are described in Table 4. These role values are + case-sensitive. + + This role field MAY be used in conjunction with the Mimetype + Section 5.1.25 to fully describe the content of the track. + + Table 4: Reserved track roles + + +==================+==========================+ + | Role | Description | + +==================+==========================+ + | audiodescription | An audio description for | + | | visually impaired users | + +------------------+--------------------------+ + | video | Visual content | + +------------------+--------------------------+ + | audio | Audio content | + +------------------+--------------------------+ + | mediatimeline | An MSF media timeline | + | | Section 7 | + +------------------+--------------------------+ + | eventtimeline | An MSF event timeline | + | | Section 8 | + +------------------+--------------------------+ + | caption | A textual representation | + | | of the audio track | + +------------------+--------------------------+ + | subtitle | A transcription of the | + | | spoken dialogue | + +------------------+--------------------------+ + | signlanguage | A visual track for | + | | hearing impaired users. | + +------------------+--------------------------+ + + Table 4 + + + +Law Expires 23 July 2026 [Page 12] + +Internet-Draft MOQT Streaming Format January 2026 + + + Custom roles MAY be used as long as they do not collide with the + specified roles. + +5.1.15. Is Live + + Location: T Required: Yes JSON Type: Boolean + + True if new Objects will be added to the track. False if no new + Objects will be added to the track. This is sent under two possible + conditions: * the publisher of a previously live track has ended the + track. * the track is Video-On-Demand (VOD) and was never live. + +5.1.16. Target latency + + Location: T Required: Optional JSON Type: Number + + The target latency in milliseconds. Target latency is defined as the + offset in wallclock time between when content was encoded and when it + is displayed to the end user. For example, if a frame of video is + encoded at 10:08:32.638 UTC and the target latency is 5000, then that + frame should be rendered to the end-user at 10:08:37.638 UTC. This + field MUST NOT be included if isLive is FALSE. All tracks belonging + to the same render group MUST have identical target latencies. All + tracks belonging to the same alternate group MUST have identical + target latencies. If this field is absent from the track definition, + then the player MAY choose the latency with which it renders the + content. + +5.1.17. Track label + + Location: T Required: Optional JSON Type: String + + A string defining a human-readable label for the track. Examples + might be "Overhead camera view" or "Deutscher Kommentar". Note that + the [JSON] spec requires UTF-8 support by decoders. + +5.1.18. Render group + + Location: T Required: Optional JSON Type: Number + + An integer specifying a group of tracks which are designed to be + rendered together. Tracks with the same group number SHOULD be + rendered simultaneously, are time-aligned and are designed to + accompany one another. A common example would be tying together + audio and video tracks. + + + + + + +Law Expires 23 July 2026 [Page 13] + +Internet-Draft MOQT Streaming Format January 2026 + + +5.1.19. Alternate group + + Location: T Required: Optional JSON Type: Number + + An integer specifying a group of tracks which are alternate versions + of one-another. Alternate tracks represent the same media content, + but differ in their selection properties. Alternate tracks MUST have + matching media time sequences. A subscriber typically subscribes to + one track from a set of tracks specifying the same alternate group + number. A common example would be a set video tracks of the same + content offered in alternate bitrates. + +5.1.20. Initialization data + + Location: T Required: Optional JSON Type: String + + A string holding Base64 [BASE64] encoded initialization data for the + track. + +5.1.21. Dependencies + + Location: T Required: Optional JSON Type: Array + + Certain tracks may depend on other tracks for decoding. Dependencies + holds an array of track names Section 5.1.11 on which the current + track is dependent. Since only the track name is signaled, the + namespace of the dependencies is assumed to match that of the track + declaring the dependencies. + +5.1.22. Temporal ID + + Location: T Required: Optional JSON Type: Number + + A number identifying the temporal layer/sub-layer encoding of the + track, starting with 0 for the base layer, and increasing by 1 for + the next higher temporal fidelity. + +5.1.23. Spatial ID + + Location: T Required: Optional JSON Type: Number + + A number identifying the spatial layer encoding of the track, + starting with 0 for the base layer, and increasing by 1 for the next + higher fidelity. + + + + + + + +Law Expires 23 July 2026 [Page 14] + +Internet-Draft MOQT Streaming Format January 2026 + + +5.1.24. Codec + + Location: T Required: Optional JSON Type: String + + A string defining the codec used to encode the track. For LOC + packaged content, the string codec registrations are defined in Sect + 3 and Section 4 of [WEBCODECS-CODEC-REGISTRY]. + +5.1.25. Mimetype + + Location: T Required: Optional JSON Type: String + + A string defining the mime type [MIME] of the track. + +5.1.26. Framerate + + Location: T Required: Optional JSON Type: Number + + A number defining the video framerate of the track, expressed as + frames per second. + +5.1.27. Timescale + + Location: T Required: Optional JSON Type: Number + + The number of time units that pass per second. + +5.1.28. Bitrate + + Location: T Required: Optional JSON Type: Number + + A number defining the bitrate of track, expressed in bits per second. + +5.1.29. Width + + Location: T Required: Optional JSON Type: Number + + A number expressing the encoded width of the video frames in pixels. + +5.1.30. Height + + Location: T Required: Optional JSON Type: Number + + A number expressing the encoded height of the video frames in pixels. + + + + + + + +Law Expires 23 July 2026 [Page 15] + +Internet-Draft MOQT Streaming Format January 2026 + + +5.1.31. Audio sample rate + + Location: T Required: Optional JSON Type: Number + + The number of audio frame samples per second. This property SHOULD + only accompany audio codecs. + +5.1.32. Channel configuration + + Location: T Required: Optional JSON Type: String + + A string specifying the audio channel configuration. This property + SHOULD only accompany audio codecs. A string is used in order to + provide the flexibility to describe complex channel configurations + for multi-channel and Next Generation Audio schemas. + +5.1.33. Display width + + Location: T Required: Optional JSON Type: Number + + A number expressing the intended display width of the track content + in pixels. + +5.1.34. Display height + + Location: T Required: Optional JSON Type: Number + + A number expressing the intended display height of the track content + in pixels. + +5.1.35. Language + + Location: T Required: Optional JSON Type: String + + A string defining the dominant language of the track. The string + MUST be one of the standard Tags for Identifying Languages as defined + by [LANG]. + +5.1.36. Parent name + + Location: T Required: Optional JSON Type: String + + A string defining the parent track name Section 5.1.11 to be cloned. + This field MUST only be included inside a Clone tracks Section 5.1.5 + object. + + + + + + +Law Expires 23 July 2026 [Page 16] + +Internet-Draft MOQT Streaming Format January 2026 + + +5.1.37. Track duration + + Location: T Required: Optional JSON Type: Number + + The duration of the track expressed in integer milliseconds. This + field MUST NOT be included if the isLive Section 5.1.15 field value + is true. + +5.2. Delta updates + + A catalog update might contain incremental changes. This is a useful + property if many tracks may be initially declared but then there are + small changes to a subset of tracks. The producer can issue a delta + update to describe these changes. Changes are described + incrementally, meaning that a delta update can itself modify a prior + delta update. + + A restricted set of operations are allowed with each delta update: * + Add a new track that has not previously been declared. * Add a new + track by cloning a previously declared track. * Remove a track that + has been previously declared. + + The following rules are to be followed in constructing and processing + delta updates: + + * A delta update MUST include the Delta Update Section 5.1.2 field + set to true. + + * A delta update catalog MUST contain at least one instance of Add + tracks Section 5.1.3, Remove tracks Section 5.1.4 or Clone Tracks + Section 5.1.5 fields and MAY contain more. It MUST NOT contain an + instance of a Tracks Section 5.1.8 field or an MSF version + Section 5.1.1 field. + + * The Add, Delete and Clone operations are applied sequentially in + the order they are declared in the document. Each operation in + the sequence is applied to the target document; the resulting + document becomes the target of the next operation. Evaluation + continues until all operations are successfully applied. + + * A Cloned track inherits all the attributes of the track defined by + the Parent Name Section 5.1.36, except the Track Name which MUST + be new. Attributes redefined in the cloning Object overwrite + inherited values. + + + + + + + +Law Expires 23 July 2026 [Page 17] + +Internet-Draft MOQT Streaming Format January 2026 + + + * The tuple of Track Namespace and Track Name defines a fixed set of + Track attributes which MUST NOT be modified after being declared. + To modify any attribute, a new track with a different + Namespace|Name tuple is created by Adding or Cloning and then the + old track is removed. + +5.3. Catalog Examples + + The following section provides non-normative JSON examples of various + catalogs compliant with this draft. + +5.3.1. Time-aligned Audio/Video Tracks with single quality + + This example shows a catalog for a media producer capable of sending + LOC packaged, time-aligned audio and video tracks. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Law Expires 23 July 2026 [Page 18] + +Internet-Draft MOQT Streaming Format January 2026 + + + { + "version": 1, + "generatedAt": 1746104606044, + "tracks": [ + { + "name": "1080p-video", + "namespace": "conference.example.com/conference123/alice", + "packaging": "loc", + "isLive": true, + "targetLatency": 2000, + "role": "video", + "renderGroup": 1, + "codec":"av01.0.08M.10.0.110.09", + "width":1920, + "height":1080, + "framerate":30, + "bitrate":1500000 + }, + { + "name": "audio", + "namespace": "conference.example.com/conference123/alice", + "packaging": "loc", + "isLive": true, + "targetLatency": 2000, + "role": "audio", + "renderGroup": 1, + "codec":"opus", + "samplerate":48000, + "channelConfig":"2", + "bitrate":32000 + } + ] + } + +5.3.2. Simulcast video tracks - 3 alternate qualities along with audio + + This example shows catalog for a media producer capable of sending 3 + time-aligned video tracks for high definition, low definition and + medium definition video qualities, along with an audio track. In + this example the namespace is absent, which infers that each track + must inherit the namespace of the catalog. + + { + "version": 1, + "generatedAt": 1746104606044, + "tracks":[ + { + "name": "hd", + + + +Law Expires 23 July 2026 [Page 19] + +Internet-Draft MOQT Streaming Format January 2026 + + + "renderGroup": 1, + "packaging": "loc", + "isLive": true, + "targetLatency": 1500, + "role": "video", + "codec":"av01", + "width":1920, + "height":1080, + "bitrate":5000000, + "framerate":30, + "altGroup":1 + }, + { + "name": "md", + "renderGroup": 1, + "packaging": "loc", + "isLive": true, + "targetLatency": 1500, + "role": "video", + "codec":"av01", + "width":720, + "height":640, + "bitrate":3000000, + "framerate":30, + "altGroup":1 + }, + { + "name": "sd", + "renderGroup": 1, + "packaging": "loc", + "isLive": true, + "targetLatency": 1500, + "role": "video", + "codec":"av01", + "width":192, + "height":144, + "bitrate":500000, + "framerate":30, + "altGroup":1 + }, + { + "name": "audio", + "renderGroup": 1, + "packaging": "loc", + "isLive": true, + "targetLatency": 1500, + "role": "audio", + "codec":"opus", + + + +Law Expires 23 July 2026 [Page 20] + +Internet-Draft MOQT Streaming Format January 2026 + + + "samplerate":48000, + "channelConfig":"2", + "bitrate":32000 + } + ] + } + +5.3.3. SVC video tracks with 2 spatial and 2 temporal qualities + + This example shows catalog for a media producer capable of sending + scalable video codec with 2 spatial and 2 temporal layers with a + dependency relation as shown below: + + +----------+ + +----------->| S1T1 | + | | 1080p30 | + | +----------+ + | ^ + | | + +----------+ | + | S1TO | | + | 1080p15 | | + +----------+ +-----+----+ + ^ | SOT1 | + | | 480p30 | + | +----------+ + | ^ + +----------+ | + | SOTO | | + | 480p15 |---------+ + +----------+ + + The corresponding catalog uses "depends" attribute to express the + track relationships. + + { + "version": 1, + "generatedAt": 1746104606044, + "tracks":[ + { + "name": "480p15", + "namespace": "conference.example.com/conference123/alice", + "renderGroup": 1, + "packaging": "loc", + "isLive": true, + "role": "video", + "codec":"av01.0.01M.10.0.110.09", + "width":640, + + + +Law Expires 23 July 2026 [Page 21] + +Internet-Draft MOQT Streaming Format January 2026 + + + "height":480, + "bitrate":3000000, + "framerate":15 + }, + { + "name": "480p30", + "namespace": "conference.example.com/conference123/alice", + "renderGroup": 1, + "packaging": "loc", + "isLive": true, + "role": "video", + "codec":"av01.0.04M.10.0.110.09", + "width":640, + "height":480, + "bitrate":3000000, + "framerate":30, + "depends": ["480p15"] + }, + { + "name": "1080p15", + "namespace": "conference.example.com/conference123/alice", + "renderGroup": 1, + "packaging": "loc", + "isLive": true, + "role": "video", + "codec":"av01.0.05M.10.0.110.09", + "width":1920, + "height":1080, + "bitrate":3000000, + "framerate":15, + "depends":["480p15"] + }, + + { + "name": "1080p30", + "namespace": "conference.example.com/conference123/alice", + "renderGroup": 1, + "packaging": "loc", + "isLive": true, + "role": "video", + "codec":"av01.0.08M.10.0.110.09", + "width":1920, + "height":1080, + "bitrate":5000000, + "framerate":30, + "depends": ["480p30", "1080p15"] + }, + { + + + +Law Expires 23 July 2026 [Page 22] + +Internet-Draft MOQT Streaming Format January 2026 + + + "name": "audio", + "namespace": "conference.example.com/conference123/alice", + "renderGroup": 1, + "packaging": "loc", + "isLive": true, + "role": "audio", + "codec":"opus", + "samplerate":48000, + "channelConfig":"2", + "bitrate":32000 + } + ] + } + +5.3.4. Delta update - adding two tracks + + This example shows the catalog delta update for a media producer + adding two tracks to an established video conference. One track is + newly declared, the other is cloned from a previous track. + + { + "deltaUpdate": true, + "generatedAt": 1746104606044, + "addTracks": [ + { + "name": "slides", + "isLive": true, + "role": "video", + "codec": "av01.0.08M.10.0.110.09", + "width": 1920, + "height": 1080, + "framerate": 15, + "bitrate": 750000, + "renderGroup": 1 + } + ], + "cloneTracks": [ + { + "parentName": "video-1080", + "name": "video-720", + "width":1280, + "height":720, + "bitrate":600000 + } + ] + } + + + + + +Law Expires 23 July 2026 [Page 23] + +Internet-Draft MOQT Streaming Format January 2026 + + +5.3.5. Delta update removing tracks + + This example shows a delta update for a media producer removing two + tracks from an established video conference. + + { + "deltaUpdate": true, + "generatedAt": 1746104606044, + "removeTracks": [{"name": "video"},{"name": "slides"}] + } + +5.3.6. Time-aligned Audio/Video Tracks with custom field values + + This example shows catalog for a media producer capable of sending + LOC packaged, time-aligned audio and video tracks along with custom + fields in each track description. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Law Expires 23 July 2026 [Page 24] + +Internet-Draft MOQT Streaming Format January 2026 + + + { + "version": 1, + "generatedAt": 1746104606044, + "tracks": [ + { + "name": "1080p-video", + "namespace": "conference.example.com/conference123/alice", + "packaging": "loc", + "isLive": true, + "role": "video", + "renderGroup": 1, + "codec":"av01.0.08M.10.0.110.09", + "width":1920, + "height":1080, + "framerate":30, + "bitrate":1500000, + "com.example-billing-code": 3201, + "com.example-tier": "premium", + "com.example-debug": "h349835bfkjfg82394d945034jsdfn349fns" + }, + { + "name": "audio", + "namespace": "conference.example.com/conference123/alice", + "packaging": "loc", + "isLive": true, + "role": "audio", + "renderGroup": 1, + "codec":"opus", + "samplerate":48000, + "channelConfig":"2", + "bitrate":32000 + } + ] + } + +5.3.7. Time-aligned VOD Audio/Video Tracks + + This example shows catalog for a media producer offering VOD (video + on-demand) non-live content. The content is LOC packaged, and + includes time-aligned audio and video tracks. + + + + + + + + + + + +Law Expires 23 July 2026 [Page 25] + +Internet-Draft MOQT Streaming Format January 2026 + + + { + "version": 1, + "tracks": [ + { + "name": "video", + "namespace": "movies.example.com/assets/boy-meets-girl-season3/episode5", + "packaging": "loc", + "isLive": false, + "trackDuration": 8072340, + "renderGroup": 1, + "codec":"av01.0.08M.10.0.110.09", + "width":1920, + "height":1080, + "framerate":30, + "bitrate":1500000 + }, + { + "name": "audio", + "namespace": "movies.example.com/assets/boy-meets-girl-season3/episode5", + "packaging": "loc", + "isLive": false, + "trackDuration": 8072340, + "renderGroup": 1, + "codec":"opus", + "samplerate":48000, + "channelConfig":"2", + "bitrate":32000 + } + ] + } + +5.3.8. Media timeline and Event timeline + + This example shows a catalog for a media producer capable of sending + LOC packaged, time-aligned audio and video tracks, along with a Media + Timeline which describes the history of those tracks and an Event + Timeline providing synchronized data. + + { + "version": 1, + "generatedAt": 1746104606044, + "tracks": [ + { + "name": "history", + "namespace": "conference.example.com/conference123/alice", + "packaging": "mediatimeline", + "mimetype": "application/json", + "depends": ["1080p-video","audio"] + + + +Law Expires 23 July 2026 [Page 26] + +Internet-Draft MOQT Streaming Format January 2026 + + + }, + { + "name": "identified-objects", + "namespace": "another-provider/time-synchronized-data", + "packaging": "eventtimeline", + "eventType": "com.ai-extraction/appID/v3", + "mimetype": "application/json", + "depends": ["1080p-video"] + }, + { + "name": "1080p-video", + "namespace": "conference.example.com/conference123/alice", + "packaging": "loc", + "isLive": true, + "targetLatency": 2000, + "role": "video", + "renderGroup": 1, + "codec":"av01.0.08M.10.0.110.09", + "width":1920, + "height":1080, + "framerate":30, + "bitrate":1500000 + }, + { + "name": "audio", + "namespace": "conference.example.com/conference123/alice", + "packaging": "loc", + "isLive": true, + "targetLatency": 2000, + "role": "audio", + "renderGroup": 1, + "codec":"opus", + "samplerate":48000, + "channelConfig":"2", + "bitrate":32000 + } + ] + } + +5.3.9. Terminating a live broadcast + + This example shows catalog for a media producer terminating a + previously live broadcast containing a video and an audio track. + + + + + + + + +Law Expires 23 July 2026 [Page 27] + +Internet-Draft MOQT Streaming Format January 2026 + + + { + "version": 1, + "generatedAt": 1746104606044, + "isComplete": true, + "tracks": [] + } + +6. Media transmission + + The MOQT Groups and MOQT Objects need to be mapped to MOQT Streams. + Irrespective of the Section 4 in place, each MOQT Object MUST be + mapped to a new MOQT Stream. + +6.1. Group numbering + + The Group ID of the first Group published in a track at application + startup MUST be a unique integer that will not repeat in the future. + One approach to achieve this is to set the initial Group ID to the + creation time of the first Object in the group, represented as the + number of milliseconds since the Unix epoch, rounded to the nearest + millisecond. This ensures that republishing the same track in the + future, such as after a loss of connectivity or an encoder restart, + will not result in smaller or duplicate Group IDs for the same track + name. Note that this method does not prevent duplication if more + than 1000 groups are published per second. + + Each subsequent Group ID MUST increase by 1. + + If a publisher is able to maintain state across a republish, it MUST + signal the gap in Group IDs using the MOQT Prior Group ID Gap + Extension header. + +7. Media Timeline track + + The media timeline track provides data about the previously published + groups and their relationship to wallclock time and media time. + Media timeline tracks allow players to seek to precise points behind + the live head in a live broadcast, or for random access in a VOD + asset. Media timeline tracks are optional. Multiple media timeline + tracks can exist inside a catalog. + +7.1. Media Timeline track payload + + A media timeline track is a JSON [JSON] document. This document MAY + be compressed using GZIP [GZIP]. The document contains an array of + records. Each record consists of an array of three required items, + whose ordinal position defines their type: + + + + +Law Expires 23 July 2026 [Page 28] + +Internet-Draft MOQT Streaming Format January 2026 + + + * The first item holds the media presentation timestamp, expressed + as a JSON Number. This value MUST match the media presentation + timestamp, rounded to the nearest millisecond, of the first media + sample in the referenced Object + + * The second item holds the MOQT Location of the entry, defined as a + tuple of the MOQT Group ID and MOQT Object ID, and expressed as a + JSON Array of Numbers, where the first number is the Group ID and + the second number is the Object ID. + + * The third time holds the wallclock time at which the media was + encoded, defined as the number of milliseconds that have elapsed + since January 1, 1970 (midnight UTC/GMT) and expressed as a JSON + Number. For VOD assets, or if the wallclock time is not known, + the value SHOULD be 0. + + An example media timeline is shown below: + + [ + [0, [0,0], 1759924158381], + [2002, [1,0], 1759924160383], + [4004, [2,0], 1759924162385], + [6006, [3,0], 1759924164387], + [8008, [4,0], 1759924166389] + ] + +7.2. Media Timeline Catalog requirements + + A media timeline track MUST carry a 'type' identifier in the Catalog + with a value of "mediatimeline". A media timeline track MUST carry a + 'depends' attribute which contains an array of all track names to + which the media timeline track applies. The mime-type of a media + timeline track MUST be specified as "application/json". + +7.3. Media Timeline track updating + + The publisher MUST publish an independent media timeline in the first + MOQT Object of each MOQT Group of a media timeline track. The + publisher MAY publish incremental updates in the second and + subsequent Objects within each Group. Incremental updates only + contain media timeline records since the last media timeline Object. + +8. Event Timeline track + + The event timeline track provides a mechanism to associate ad-hoc + event metadata with the broadcast. Use-case examples include live + sports score data, GPS coordinates of race cars, SAP-types for media + segments or active speaker notifications in web conferences. + + + +Law Expires 23 July 2026 [Page 29] + +Internet-Draft MOQT Streaming Format January 2026 + + + To allow the client to bind this event metadata with the broadcast + content described by the media timeline track, each event record MUST + contain a reference to one of Media PTS, wallclock time or MOQT + Location. + + Event timeline tracks are optional. Multiple event timeline tracks + can exist inside a catalog. The type & structure of the data + contained within each event timeline track is declared in the + catalog, to facilitate client selection and parsing. + +8.1. Event Timeline data format + + An event timeline track is a JSON [JSON] document. This document MAY + be compressed using GZIP [GZIP]. The document contains an array of + records. Each record consists of a JSON Object containing the + following required fields: + + * An index reference, which MUST be either 't' for wallclock time, + 'l' for Location or 'm' for Media PTS. Only one of these index + values may be used within each record. Event timelines SHOULD use + the same index reference type for each record. The definitions + for wallclock time, Location and Media PTS are identical to those + defined for media timeline payload Section 7.1. Wallclock time + and media PTS values are JSON Number, while Location value is an + Array of Numbers, where the first item represents the MOQT GroupID + and the second item the MOQT Object ID. + + * A 'data' Object, whose structure is defined by the Section 5.1.13 + value declared for this track in the Catalog. + +8.2. Event Timeline Catalog requirements + + An event timeline track MUST carry: + + * a Section 5.1.12 attribute with a value of "eventtimeline". + + * a Section 5.1.21 attribute which contains an array of all track + names to which the event timeline track applies. + + * a Section 5.1.25 attribute with a value of "application/json". + + * an Section 5.1.13 attribute declaring the type & structure of data + contained in the event timeline track. + + + + + + + + +Law Expires 23 July 2026 [Page 30] + +Internet-Draft MOQT Streaming Format January 2026 + + +8.3. Event Timeline track updating + + The publisher MUST publish an independent event timeline in the first + MOQT Object of each MOQT Group of an event timeline track. The + publisher MAY publish incremental updates in the second and + subsequent Objects within each Group. Incremental updates only + contain event timeline records since the last event timeline Object. + +8.4. Event timeline track examples + +8.4.1. Event timeline track with wallclock time indexing + + This example shows how sports scores and game information might be + defined in a live sports broadcast. + + [ + { + "t": 1756885678361, + "data": { + "status": "in_progress", + "period": 1, + "clock": "12:00", + "homeScore": 0, + "awayScore": 0, + "lastPlay": "Game Start" + } + }, + { + "t": 1756885981542, + "data": { + "status": "in_progress", + "period": 1, + "clock": "09:25", + "homeScore": 2, + "awayScore": 0, + "lastPlay": "Team A: #23 makes 2-pt jump shot" + } + } + ] + +8.4.2. Event timeline track with MOQT Location indexing + + This example shows drone GPS coordinates synched with the start of + each Group. + + + + + + + +Law Expires 23 July 2026 [Page 31] + +Internet-Draft MOQT Streaming Format January 2026 + + + [ + { + "l": [0,0], + "data": [47.1812,8.4592] + }, + { + "l": [1,0], + "data": [47.1662,8.5155] + } + ] + +9. Workflow + +9.1. Initiating a broadcast + + An MSF publisher MUST publish a catalog track object before + publishing any media track objects. + +9.2. Ending a live broadcast + + After publishing a catalog and defining tracks carrying live content, + an original publisher can deliver a deterministic signal to all + subscribers that the broadcast is complete by taking the following + steps: + + * Send a SUBSCRIBE_DONE (See MOQT Sect 8.1.2) message for all active + tracks using status code 0x2 Track Ended. + + * If the live stream is being converted instantly to a VOD asset, + then publish an independent (non-delta) catalog update which, for + each track, sets isLive Section 5.1.15 to FALSE and adds a track + duration Section 5.1.37 field. + + * If the live stream is being terminated permanently without + conversion to VOD, then publish an independent catalog update + which signals isComplete Section 5.1.7 as TRUE and which contains + an empty Tracks Section 5.1.8 field. + +10. Security Considerations + + ToDo + +11. IANA Considerations + + This document creates a new entry in the "MoQ Streaming Format" + Registry (see [MoQTransport] Sect 8). The type value is 0x001, the + name is "MOQT Streaming Format" and the RFC is XXX. + + + + +Law Expires 23 July 2026 [Page 32] + +Internet-Draft MOQT Streaming Format January 2026 + + +12. Normative References + + [BASE64] Josefsson, S., "The Base16, Base32, and Base64 Data + Encodings", RFC 4648, DOI 10.17487/RFC4648, October 2006, + . + + [GZIP] Deutsch, P., "GZIP file format specification version 4.3", + RFC 1952, DOI 10.17487/RFC1952, May 1996, + . + + [IMSC1] "W3C, TTML Profiles for Internet Media Subtitles and + Captions 1.0 (IMSC1)", April 2016, + . + + [JSON] Bray, T., Ed., "The JavaScript Object Notation (JSON) Data + Interchange Format", STD 90, RFC 8259, + DOI 10.17487/RFC8259, December 2017, + . + + [LANG] Phillips, A., Ed. and M. Davis, Ed., "Tags for Identifying + Languages", BCP 47, RFC 5646, DOI 10.17487/RFC5646, + September 2009, . + + [LOC] Zanaty, M., Nandakumar, S., and P. Thatcher, "Low Overhead + Media Container", Work in Progress, Internet-Draft, draft- + mzanaty-moq-loc-05, 3 March 2025, + . + + [MIME] Freed, N., Klensin, J., and T. Hansen, "Media Type + Specifications and Registration Procedures", BCP 13, + RFC 6838, DOI 10.17487/RFC6838, January 2013, + . + + [MoQTransport] + Nandakumar, S., Vasiliev, V., Swett, I., and A. Frindell, + "Media over QUIC Transport", Work in Progress, Internet- + Draft, draft-ietf-moq-transport-11, 28 April 2025, + . + + [RFC2119] Bradner, S., "Key words for use in RFCs to Indicate + Requirement Levels", BCP 14, RFC 2119, + DOI 10.17487/RFC2119, March 1997, + . + + + + + + +Law Expires 23 July 2026 [Page 33] + +Internet-Draft MOQT Streaming Format January 2026 + + + [RFC4180] Shafranovich, Y., "Common Format and MIME Type for Comma- + Separated Values (CSV) Files", RFC 4180, + DOI 10.17487/RFC4180, October 2005, + . + + [RFC8174] Leiba, B., "Ambiguity of Uppercase vs Lowercase in RFC + 2119 Key Words", BCP 14, RFC 8174, DOI 10.17487/RFC8174, + May 2017, . + + [RFC9000] Iyengar, J., Ed. and M. Thomson, Ed., "QUIC: A UDP-Based + Multiplexed and Secure Transport", RFC 9000, + DOI 10.17487/RFC9000, May 2021, + . + + [WEBCODECS-CODEC-REGISTRY] + "WebCodecs Codec Registry", September 2024, + . + + [WEBVTT] "World Wide Web Consortium (W3C), WebVTT: The Web Video + Text Tracks Format", April 2019, + . + +Acknowledgments + + * the MoQ Workgroup and mailing lists. + +Contributors + + The following persons where the co-authors of the individual draft + (draft-law-moq-warpstreamingformat) this document is based on: + + * Luke Curley + + * Victor Vasiliev + + * Suhas Nandakumar + + * Kirill Pugin + + * Will Law + +Author's Address + + Will Law + Akamai + Email: wilaw@akamai.com + + + + + +Law Expires 23 July 2026 [Page 34] diff --git a/js/hang/src/catalog/container.ts b/js/hang/src/catalog/container.ts index 3d289f318..f931afdd5 100644 --- a/js/hang/src/catalog/container.ts +++ b/js/hang/src/catalog/container.ts @@ -1,5 +1,4 @@ import { z } from "zod"; -import { u53Schema } from "./integers"; /** * Container format for frame timestamp encoding and frame payload structure. @@ -7,19 +6,17 @@ import { u53Schema } from "./integers"; * - "legacy": Uses QUIC VarInt encoding (1-8 bytes, variable length), raw frame payloads. * Timestamps are in microseconds. * - "cmaf": Fragmented MP4 container - frames contain complete moof+mdat fragments. - * Timestamps are in timescale units. + * The init segment (ftyp+moov) is base64-encoded in the catalog. */ export const ContainerSchema = z .discriminatedUnion("kind", [ // The default hang container z.object({ kind: z.literal("legacy") }), - // CMAF container with timescale for timestamp conversion + // CMAF container with base64-encoded init segment (ftyp+moov) z.object({ kind: z.literal("cmaf"), - // Time units per second - timescale: u53Schema, - // Track ID used in the moof/mdat fragments - trackId: u53Schema, + // Base64-encoded init segment (ftyp+moov) + initData: z.string().base64(), }), ]) .default({ kind: "legacy" }); diff --git a/js/hang/src/container/cmaf/encode.ts b/js/hang/src/container/cmaf/encode.ts index 104cb4b98..923b6bae3 100644 --- a/js/hang/src/container/cmaf/encode.ts +++ b/js/hang/src/container/cmaf/encode.ts @@ -232,17 +232,14 @@ function createAvc1Box(width: number, height: number, avcC: Uint8Array): Uint8Ar * ``` */ export function createVideoInitSegment(config: Catalog.VideoConfig): Uint8Array { - const { codedWidth, codedHeight, description, container } = config; + const { codedWidth, codedHeight, description } = config; if (!codedWidth || !codedHeight || !description) { - // TODO: We could throw new Error("Missing required fields to create video init segment"); } - // Use timescale from CMAF container, or microseconds for legacy - const timescale = container.kind === "cmaf" ? container.timescale : 1_000_000; - - // Use track_id from CMAF container, or default to 1 for legacy - const trackId = container.kind === "cmaf" ? container.trackId : 1; + // Legacy container always uses microsecond timescale and track ID 1 + const timescale = 1_000_000; + const trackId = 1; // ftyp - File Type Box const ftyp: FileTypeBox = { @@ -442,13 +439,11 @@ export function createVideoInitSegment(config: Catalog.VideoConfig): Uint8Array * Supports AAC (mp4a) and Opus codecs. */ export function createAudioInitSegment(config: Catalog.AudioConfig): Uint8Array { - const { sampleRate, numberOfChannels, description, codec, container } = config; - - // Use timescale from CMAF container, or microseconds for legacy - const timescale = container.kind === "cmaf" ? container.timescale : 1_000_000; + const { sampleRate, numberOfChannels, description, codec } = config; - // Use track_id from CMAF container, or default to 1 for legacy - const trackId = container.kind === "cmaf" ? container.trackId : 1; + // Legacy container always uses microsecond timescale and track ID 1 + const timescale = 1_000_000; + const trackId = 1; // ftyp - File Type Box const ftyp: FileTypeBox = { diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 3631b8bcd..411f29d1a 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -244,7 +244,9 @@ export class Decoder { #runCmafDecoder(effect: Effect, sub: Moq.Track, config: Catalog.AudioConfig): void { if (config.container.kind !== "cmaf") return; // just to help typescript - const { timescale } = config.container; + // Decode the base64 init segment to extract timescale + const initBytes = Uint8Array.from(atob(config.container.initData), (c) => c.charCodeAt(0)); + const { timescale } = Container.Cmaf.decodeInitSegment(initBytes); const description = config.description ? Util.Hex.toBytes(config.description) : undefined; // For CMAF, just use decode buffer (no network jitter buffer yet) diff --git a/js/watch/src/audio/mse.ts b/js/watch/src/audio/mse.ts index 8b4ccb309..89cd8e00c 100644 --- a/js/watch/src/audio/mse.ts +++ b/js/watch/src/audio/mse.ts @@ -103,12 +103,13 @@ export class Mse implements Backend { ): void { if (config.container.kind !== "cmaf") throw new Error("unreachable"); - const timescale = config.container.timescale; + // Decode the base64 init segment from the catalog + const initBytes = Uint8Array.from(atob(config.container.initData), (c) => c.charCodeAt(0)); + const { timescale } = Container.Cmaf.decodeInitSegment(initBytes); effect.spawn(async () => { - // Generate init segment from catalog config (uses track_id from container) - const initSegment = Container.Cmaf.createAudioInitSegment(config); - await this.#appendBuffer(sourceBuffer, initSegment); + // Append the passthrough init segment directly + await this.#appendBuffer(sourceBuffer, initBytes); for (;;) { // TODO: Use Frame.Consumer for CMAF so we can support higher latencies. diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index 0f00f9811..9d42dad3c 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -355,7 +355,9 @@ class DecoderTrack { #runCmaf(effect: Effect, sub: Moq.Track, decoder: VideoDecoder): void { if (this.config.container.kind !== "cmaf") return; - const { timescale } = this.config.container; + // Decode the base64 init segment to extract timescale + const initBytes = Uint8Array.from(atob(this.config.container.initData), (c) => c.charCodeAt(0)); + const { timescale } = Container.Cmaf.decodeInitSegment(initBytes); const description = this.config.description ? Util.Hex.toBytes(this.config.description) : undefined; // Configure decoder with description from catalog diff --git a/js/watch/src/video/mse.ts b/js/watch/src/video/mse.ts index 50cdc9a68..88f3420a7 100644 --- a/js/watch/src/video/mse.ts +++ b/js/watch/src/video/mse.ts @@ -107,12 +107,13 @@ export class Mse implements Backend { const data = active.subscribe(track, Catalog.PRIORITY.video); effect.cleanup(() => data.close()); - const timescale = config.container.timescale; + // Decode the base64 init segment from the catalog + const initBytes = Uint8Array.from(atob(config.container.initData), (c) => c.charCodeAt(0)); + const { timescale } = Container.Cmaf.decodeInitSegment(initBytes); effect.spawn(async () => { - // Generate init segment from catalog config (uses track_id from container) - const initSegment = Container.Cmaf.createVideoInitSegment(config); - await this.#appendBuffer(sourceBuffer, initSegment); + // Append the passthrough init segment directly + await this.#appendBuffer(sourceBuffer, initBytes); for (;;) { // TODO: Use Frame.Consumer for CMAF so we can support higher latencies. diff --git a/rs/hang/src/catalog/container.rs b/rs/hang/src/catalog/container.rs index ea7a416d5..1b84684e4 100644 --- a/rs/hang/src/catalog/container.rs +++ b/rs/hang/src/catalog/container.rs @@ -5,11 +5,11 @@ use serde::{Deserialize, Serialize}; /// - "legacy": Uses QUIC VarInt encoding (1-8 bytes, variable length), raw frame payloads. /// Timestamps are in microseconds. /// - "cmaf": Fragmented MP4 container - frames contain complete moof+mdat fragments. -/// Timestamps are in timescale units. +/// The init segment (ftyp+moov) is base64-encoded in the catalog. /// /// JSON example: /// ```json -/// { "kind": "cmaf", "timescale": 1000000, "trackId": 1 } +/// { "kind": "cmaf", "initData": "" } /// ``` #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Default)] #[serde(rename_all = "camelCase")] @@ -19,10 +19,7 @@ pub enum Container { #[default] Legacy, Cmaf { - /// Time units per second - timescale: u64, - /// Track ID used in the moof/mdat fragments - #[serde(rename = "trackId")] - track_id: u32, + /// Base64-encoded init segment (ftyp+moov) + init_data: String, }, } diff --git a/rs/moq-mux/src/import/fmp4.rs b/rs/moq-mux/src/import/fmp4.rs index 9a7574190..13ff56bc8 100644 --- a/rs/moq-mux/src/import/fmp4.rs +++ b/rs/moq-mux/src/import/fmp4.rs @@ -1,8 +1,9 @@ use anyhow::Context; +use base64::Engine; use bytes::{Buf, Bytes, BytesMut}; use hang::catalog::{AAC, AV1, AudioCodec, AudioConfig, Container, H264, H265, VP9, VideoCodec, VideoConfig}; use hang::container::Timestamp; -use mp4_atom::{Any, Atom, DecodeMaybe, Mdat, Moof, Moov, Trak}; +use mp4_atom::{Any, Atom, DecodeMaybe, Encode, Mdat, Moof, Moov, Trak}; use std::collections::HashMap; use tokio::io::{AsyncRead, AsyncReadExt}; @@ -186,12 +187,12 @@ impl Fmp4 { let (kind, track) = match handler.as_ref() { b"vide" => { - let config = self.init_video(trak)?; + let config = self.init_video(trak, &moov)?; let track = catalog.video.create_track(ext, config.clone()); (TrackKind::Video, track) } b"soun" => { - let config = self.init_audio(trak)?; + let config = self.init_audio(trak, &moov)?; let track = catalog.audio.create_track(ext, config.clone()); (TrackKind::Audio, track) } @@ -228,19 +229,52 @@ impl Fmp4 { Ok(()) } - fn container(&self, trak: &Trak) -> Container { + fn container(&self, trak: &Trak, moov: &Moov) -> anyhow::Result { if self.config.passthrough { - Container::Cmaf { - timescale: trak.mdia.mdhd.timescale as u64, - track_id: trak.tkhd.track_id, - } + // Build a single-track init segment (ftyp+moov) for this track. + let ftyp = mp4_atom::Ftyp { + major_brand: b"isom".into(), + minor_version: 0x200, + compatible_brands: vec![b"isom".into(), b"iso6".into(), b"mp41".into()], + }; + + // Build a moov with just this single track and matching mvex/trex. + let track_id = trak.tkhd.track_id; + let trex = moov + .mvex + .as_ref() + .and_then(|mvex| mvex.trex.iter().find(|trex| trex.track_id == track_id)) + .cloned() + .unwrap_or(mp4_atom::Trex { + track_id, + default_sample_description_index: 1, + ..Default::default() + }); + + let single_moov = Moov { + mvhd: moov.mvhd.clone(), + trak: vec![trak.clone()], + mvex: Some(mp4_atom::Mvex { + mehd: None, + trex: vec![trex], + }), + meta: None, + udta: None, + }; + + let mut buf = Vec::new(); + ftyp.encode(&mut buf)?; + single_moov.encode(&mut buf)?; + + let init_data = base64::engine::general_purpose::STANDARD.encode(&buf); + Ok(Container::Cmaf { init_data }) } else { - Container::Legacy + Ok(Container::Legacy) } } - fn init_video(&mut self, trak: &Trak) -> anyhow::Result { - let container = self.container(trak); + fn init_video(&mut self, trak: &Trak, moov: &Moov) -> anyhow::Result { + let container = self.container(trak, moov)?; let stsd = &trak.mdia.minf.stbl.stsd; let codec = match stsd.codecs.len() { @@ -363,7 +397,6 @@ impl Fmp4 { Ok(config) } - // There's two almost identical hvcc atoms in the wild. fn init_h265( &mut self, in_band: bool, @@ -399,8 +432,8 @@ impl Fmp4 { }) } - fn init_audio(&mut self, trak: &Trak) -> anyhow::Result { - let container = self.container(trak); + fn init_audio(&mut self, trak: &Trak, moov: &Moov) -> anyhow::Result { + let container = self.container(trak, moov)?; let stsd = &trak.mdia.minf.stbl.stsd; let codec = match stsd.codecs.len() { diff --git a/rs/moq-mux/src/msf.rs b/rs/moq-mux/src/msf.rs index a924d2b3e..37f68b83e 100644 --- a/rs/moq-mux/src/msf.rs +++ b/rs/moq-mux/src/msf.rs @@ -12,15 +12,16 @@ pub fn to_msf(catalog: &hang::Catalog) -> moq_msf::Catalog { for (name, config) in &catalog.video.renditions { let packaging = match &config.container { hang::catalog::Container::Cmaf { .. } => moq_msf::Packaging::Cmaf, - // TODO: For CMAF packaging, build proper CMAF init segments (ftyp+moov). - // See draft-ietf-moq-cmsf-00 for the required structure. _ => moq_msf::Packaging::Legacy, }; - let init_data = config - .description - .as_ref() - .map(|d| base64::engine::general_purpose::STANDARD.encode(d.as_ref())); + let init_data = match &config.container { + hang::catalog::Container::Cmaf { init_data } => Some(init_data.clone()), + _ => config + .description + .as_ref() + .map(|d| base64::engine::general_purpose::STANDARD.encode(d.as_ref())), + }; tracks.push(moq_msf::Track { name: name.clone(), @@ -47,10 +48,13 @@ pub fn to_msf(catalog: &hang::Catalog) -> moq_msf::Catalog { _ => moq_msf::Packaging::Legacy, }; - let init_data = config - .description - .as_ref() - .map(|d| base64::engine::general_purpose::STANDARD.encode(d.as_ref())); + let init_data = match &config.container { + hang::catalog::Container::Cmaf { init_data } => Some(init_data.clone()), + _ => config + .description + .as_ref() + .map(|d| base64::engine::general_purpose::STANDARD.encode(d.as_ref())), + }; tracks.push(moq_msf::Track { name: name.clone(), @@ -242,8 +246,7 @@ mod test { framerate: None, optimize_for_latency: None, container: Container::Cmaf { - timescale: 90000, - track_id: 1, + init_data: "AAAYZ2Z0eXA=".to_string(), }, jitter: None, }, @@ -262,5 +265,6 @@ mod test { let msf = to_msf(&catalog); let video = &msf.tracks[0]; assert_eq!(video.packaging, moq_msf::Packaging::Cmaf); + assert_eq!(video.init_data, Some("AAAYZ2Z0eXA=".to_string())); } } From 81ce43f42d074256c02c52b9faaf42956a2852cf Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 6 Mar 2026 15:31:57 -0800 Subject: [PATCH 02/25] Add unit tests for fMP4 importer init segment generation (#1066) Co-authored-by: Claude Opus 4.6 --- .gitignore | 10 -- dev/.gitignore | 9 ++ rs/moq-mux/src/catalog.rs | 5 + rs/moq-mux/src/import/mod.rs | 3 + rs/moq-mux/src/import/test/av1.mp4 | Bin 0 -> 1264 bytes rs/moq-mux/src/import/test/bbb.mp4 | Bin 0 -> 2470 bytes rs/moq-mux/src/import/test/mod.rs | 157 +++++++++++++++++++++++++++++ rs/moq-mux/src/import/test/vp9.mp4 | Bin 0 -> 1906 bytes 8 files changed, 174 insertions(+), 10 deletions(-) create mode 100644 dev/.gitignore create mode 100644 rs/moq-mux/src/import/test/av1.mp4 create mode 100644 rs/moq-mux/src/import/test/bbb.mp4 create mode 100644 rs/moq-mux/src/import/test/mod.rs create mode 100644 rs/moq-mux/src/import/test/vp9.mp4 diff --git a/.gitignore b/.gitignore index 41d6afd70..07cefbb37 100644 --- a/.gitignore +++ b/.gitignore @@ -19,16 +19,6 @@ result node_modules /.direnv -# Don't leak dev stuff -*.mp4 -*.fmp4 -*.crt -*.key -*.hex -*.jwk -*.jwt -*.m3u8 -*.m4s # We're using bun package-lock.json diff --git a/dev/.gitignore b/dev/.gitignore new file mode 100644 index 000000000..c6c8158db --- /dev/null +++ b/dev/.gitignore @@ -0,0 +1,9 @@ +*.mp4 +*.fmp4 +*.crt +*.key +*.hex +*.jwk +*.jwt +*.m3u8 +*.m4s diff --git a/rs/moq-mux/src/catalog.rs b/rs/moq-mux/src/catalog.rs index dcf6fecf4..a99b85a02 100644 --- a/rs/moq-mux/src/catalog.rs +++ b/rs/moq-mux/src/catalog.rs @@ -51,6 +51,11 @@ impl CatalogProducer { } } + /// Get a snapshot of the current catalog. + pub fn snapshot(&self) -> hang::Catalog { + self.current.lock().unwrap().clone() + } + /// Create a consumer for this catalog, receiving updates as they're published. pub fn consume(&self) -> hang::CatalogConsumer { hang::CatalogConsumer::new(self.hang_track.consume()) diff --git a/rs/moq-mux/src/import/mod.rs b/rs/moq-mux/src/import/mod.rs index e0fedd086..cc1073cb9 100644 --- a/rs/moq-mux/src/import/mod.rs +++ b/rs/moq-mux/src/import/mod.rs @@ -43,3 +43,6 @@ pub use hev1::*; #[cfg(feature = "hls")] pub use hls::*; pub use opus::*; + +#[cfg(test)] +mod test; diff --git a/rs/moq-mux/src/import/test/av1.mp4 b/rs/moq-mux/src/import/test/av1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fd0272dbdc761d0612058615d3de7a2abe3ff52b GIT binary patch literal 1264 zcmcgr&1(};5T9)$H8zDP0b7YQLctzP$fgn#FRs*TkrHyKLPgL`b~jnNuN!t>wtie9 z@u-MM+dn`m6))mN@S>nzJm|r@RS@)0q(~2~igD(pAKMsvbjW-2`_25`$IiR}K+ND* zE2e9w0Xl$ukr_qROUkTrRL0mUvuzImEasIp^giqSDu@Jtzb)HFFx`~2q(|f z^bJhjE>x8>`IEz`gd9ILaz5_po}A6bOBp?tHPU*f5HFd$TrFH;s>^lSM(Stglz|}P zF>$W{mIGQd7u6h{G-Q+yRUIp~ju%~Cun^yKxx3L1($uME=*j$hia=Klzj_oQ4YI!! z&9HCoZ8VVesh&K1BRB?ox{;lI^!(*?;|nd06u85Myk|uL8USIJb63Qg6LPoc;}PW7 ze4HR^incE{$UOZD@)&o7rxcyndSSJyaTVheOc$PXhy1O=_Bv{%LZ#ZYBc|ojr3W5z zn+}a?s#q*GA5%ReEhmm82m1fPBi9Cn802lw4tWB+^IKd)K8 z^C3D@|J``Dm*4&VcJ%Z7w~5>PPeU+Qx{O6Ax2ZRkC%<#OrPuB2EqW1IuQ%12M&%DJ C8}{P> literal 0 HcmV?d00001 diff --git a/rs/moq-mux/src/import/test/bbb.mp4 b/rs/moq-mux/src/import/test/bbb.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1c42daf3f22d346edbfffef44c10e10471446739 GIT binary patch literal 2470 zcmeHHO=uHA6n?v#C}Pl5QfXdMK#4%_dD~HXC*)>)#~@ z1&{t*{JD4)i+^|!^iV`p^kOOKQF{{+K~NAx<9oB4WSeeE6}>sh?E9TJ^WOVr0H8h1 zP8PD3(GMU3(WIVE>V<7_fS{xsh7ABY-Ogx8JwtlW1pxs6wXr}zb)Br#JpDKk&(O?F zbqbA#*%aSX@;KH-Tf$Q3si12Rr}&E22zPZYt0E)oT76zeX2|2v6Z*ogjFvMwhaWqu zr7AqpO+%aI-dJphJZu<~xs*JbPex@sHV}>XMM)n;A7NO}=F`YTY@P4qJy*Bp*xEHS zMLsmh9x9qS+2Mti7MsW+|IlJqr32)1Q?--WAwIk@xgtn;jvz~u#v%_#?fJHp%P7S$ z>X4v8M|mDvCy`EfpAnSx9|H=21o#PcJq=wHAei~o`}__r2p3eVfRr|%8JS=vLkh5A z;y9n^V=-oNo%2E2N_yl7%BMUsvFMXDYC2(d2~Z$t77c-CT%UHGtSne2=#4=!y_;P9 zKw%lhyf@6UKq0xH2wM&)Dgl-O2zcfK;&dG#rtr$DZlGG8B7{ljoFDp9mc~0Fl}e?J zc-;d?BT6&52p=xd45CI{7hZuxpu*M~>N=qAFEkWJbUSqdW68|q6xk|>h_4KGnE+ zW?1Ex$FGzTx0F;yn)DboQ5id7^t{*#T_sfbZH(pn(~bF|8gDEwy)mu!%h=6;gQv$I zJly8D@m3c&U5TrKHA90xJ~!;~iZ2c7L0I%VmfHxl0+jt55h9!4vHstkhJE;r8>b|G E0a5Ef-~a#s literal 0 HcmV?d00001 diff --git a/rs/moq-mux/src/import/test/mod.rs b/rs/moq-mux/src/import/test/mod.rs new file mode 100644 index 000000000..072ff6246 --- /dev/null +++ b/rs/moq-mux/src/import/test/mod.rs @@ -0,0 +1,157 @@ +use base64::Engine; +use hang::catalog::Container; +use mp4_atom::{Decode, Encode}; + +fn run_fmp4(data: &[u8], passthrough: bool) -> hang::Catalog { + let mut broadcast = moq_lite::BroadcastProducer::new(); + let catalog = crate::CatalogProducer::new(&mut broadcast).unwrap(); + + let config = super::Fmp4Config { passthrough }; + let mut fmp4 = super::Fmp4::new(broadcast, catalog.clone(), config); + + let mut buf = bytes::BytesMut::from(data); + // Ignore errors from incomplete/malformed trailing fragments in test files. + let _ = fmp4.decode(&mut buf); + + catalog.snapshot() +} + +fn decode_init_data(init_data: &str) -> (mp4_atom::Ftyp, mp4_atom::Moov) { + let bytes = base64::engine::general_purpose::STANDARD + .decode(init_data) + .expect("invalid base64"); + let mut cursor = std::io::Cursor::new(&bytes); + let ftyp = mp4_atom::Ftyp::decode(&mut cursor).expect("invalid ftyp"); + let moov = mp4_atom::Moov::decode(&mut cursor).expect("invalid moov"); + (ftyp, moov) +} + +#[test] +fn test_bbb_passthrough_catalog() { + let data = include_bytes!("bbb.mp4"); + let catalog = run_fmp4(data, true); + + assert_eq!(catalog.video.renditions.len(), 1); + assert_eq!(catalog.audio.renditions.len(), 1); + + let video = catalog.video.renditions.values().next().unwrap(); + assert_eq!(video.codec.to_string(), "avc1.64001f"); + assert_eq!(video.coded_width, Some(1280)); + assert_eq!(video.coded_height, Some(720)); + assert!(matches!(video.container, Container::Cmaf { .. })); + + let audio = catalog.audio.renditions.values().next().unwrap(); + assert_eq!(audio.codec.to_string(), "mp4a.40.2"); + assert_eq!(audio.sample_rate, 44100); + assert_eq!(audio.channel_count, 2); + assert!(matches!(audio.container, Container::Cmaf { .. })); +} + +#[test] +fn test_bbb_passthrough_init_data_roundtrip() { + let data = include_bytes!("bbb.mp4"); + let catalog = run_fmp4(data, true); + + // Check video init data + let video = catalog.video.renditions.values().next().unwrap(); + let Container::Cmaf { init_data } = &video.container else { + panic!("expected Cmaf container"); + }; + let (ftyp, moov) = decode_init_data(init_data); + assert_eq!(ftyp.major_brand, mp4_atom::FourCC::new(b"isom")); + assert_eq!(moov.trak.len(), 1); + assert_eq!(moov.trak[0].tkhd.track_id, 1); + assert_eq!(moov.trak[0].mdia.mdhd.timescale, 24000); + let mvex = moov.mvex.as_ref().unwrap(); + assert_eq!(mvex.trex.len(), 1); + assert_eq!(mvex.trex[0].track_id, 1); + + // Verify it round-trips through encode/decode + let mut buf = Vec::new(); + ftyp.encode(&mut buf).unwrap(); + moov.encode(&mut buf).unwrap(); + let (ftyp2, moov2) = decode_init_data(&base64::engine::general_purpose::STANDARD.encode(&buf)); + assert_eq!(ftyp2.major_brand, mp4_atom::FourCC::new(b"isom")); + assert_eq!(moov2.trak.len(), 1); + + // Check audio init data + let audio = catalog.audio.renditions.values().next().unwrap(); + let Container::Cmaf { init_data } = &audio.container else { + panic!("expected Cmaf container"); + }; + let (ftyp, moov) = decode_init_data(init_data); + assert_eq!(ftyp.major_brand, mp4_atom::FourCC::new(b"isom")); + assert_eq!(moov.trak.len(), 1); + assert_eq!(moov.trak[0].tkhd.track_id, 2); + assert_eq!(moov.trak[0].mdia.mdhd.timescale, 44100); + let mvex = moov.mvex.as_ref().unwrap(); + assert_eq!(mvex.trex.len(), 1); + assert_eq!(mvex.trex[0].track_id, 2); +} + +#[test] +fn test_bbb_legacy_catalog() { + let data = include_bytes!("bbb.mp4"); + let catalog = run_fmp4(data, false); + + assert_eq!(catalog.video.renditions.len(), 1); + assert_eq!(catalog.audio.renditions.len(), 1); + + let video = catalog.video.renditions.values().next().unwrap(); + assert_eq!(video.codec.to_string(), "avc1.64001f"); + assert_eq!(video.coded_width, Some(1280)); + assert_eq!(video.coded_height, Some(720)); + assert!(matches!(video.container, Container::Legacy)); + + let audio = catalog.audio.renditions.values().next().unwrap(); + assert_eq!(audio.codec.to_string(), "mp4a.40.2"); + assert_eq!(audio.sample_rate, 44100); + assert_eq!(audio.channel_count, 2); + assert!(matches!(audio.container, Container::Legacy)); +} + +#[test] +fn test_av1_passthrough_catalog() { + let data = include_bytes!("av1.mp4"); + let catalog = run_fmp4(data, true); + + assert_eq!(catalog.video.renditions.len(), 1); + assert_eq!(catalog.audio.renditions.len(), 0); + + let video = catalog.video.renditions.values().next().unwrap(); + assert!(video.codec.to_string().starts_with("av01."), "codec: {}", video.codec); + assert!(matches!(video.container, Container::Cmaf { .. })); + + let Container::Cmaf { init_data } = &video.container else { + panic!("expected Cmaf container"); + }; + let (ftyp, moov) = decode_init_data(init_data); + assert_eq!(ftyp.major_brand, mp4_atom::FourCC::new(b"isom")); + assert_eq!(moov.trak.len(), 1); + let mvex = moov.mvex.as_ref().unwrap(); + assert_eq!(mvex.trex.len(), 1); + assert_eq!(mvex.trex[0].track_id, moov.trak[0].tkhd.track_id); +} + +#[test] +fn test_vp9_passthrough_catalog() { + let data = include_bytes!("vp9.mp4"); + let catalog = run_fmp4(data, true); + + assert_eq!(catalog.video.renditions.len(), 1); + assert_eq!(catalog.audio.renditions.len(), 0); + + let video = catalog.video.renditions.values().next().unwrap(); + assert!(video.codec.to_string().starts_with("vp09."), "codec: {}", video.codec); + assert!(matches!(video.container, Container::Cmaf { .. })); + + let Container::Cmaf { init_data } = &video.container else { + panic!("expected Cmaf container"); + }; + let (ftyp, moov) = decode_init_data(init_data); + assert_eq!(ftyp.major_brand, mp4_atom::FourCC::new(b"isom")); + assert_eq!(moov.trak.len(), 1); + let mvex = moov.mvex.as_ref().unwrap(); + assert_eq!(mvex.trex.len(), 1); + assert_eq!(mvex.trex[0].track_id, moov.trak[0].tkhd.track_id); +} diff --git a/rs/moq-mux/src/import/test/vp9.mp4 b/rs/moq-mux/src/import/test/vp9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..12a6ae236a56ed519563a12fd5c5966dacea6bcb GIT binary patch literal 1906 zcmZvdO=uHA6vt<0HU^5+fDwy`1TBJKZEXE$JO$H3dn*NdsVLj*=EG#WAv=j}K`BKP zZ-O8wc=D#`Mb8#Lu!0`Ecu>KEASx88J&Bk4-s~ptSv&3S{@-u@^LFOVED=KVm-Kw2 z8V0_A2G0?{aqOt;gq3z9n=AUIq7ZW34}vyy*Zp?IMc?~_PwwL-cAU;*EG+YHWGAK| z>gREu2mfmc=$h|oC+^Br+gf16-C?^{K-q>CCML(HK;?v}Xx#`;4Gxy8 zy3(3CQVjgTauAg3-e6d9YR*8zDb}2_*A&M~#a#B}iBfK8!1ZQYWmwd*ZaOtcb5_^P z(v#h#Z(4CYmFNNG_@`}(^U!JGZ4EAAP5W*Ymzwe2t;>7nCHlkwFGIWPdWOcgDTsqH zXV$I5LLlijzdBn2(eFmjy>8RP7#mJ!d#hQ`#QIgJXX>CALLHj@jrj#U1E8^OS^|JP zOn(e|@WN*GOidIrg#ca%mD{TIZEOHew;RQRN!PO^CGNElsZi_CsOSc92q&-G(hx6n zqey_K7%-M@`=)$+rXzman*&eE_oC8@L(7X0XLQqW9FBgt7QbTRzX^mXMU-w-irAw|h!_7^t>Sk+Jh$PNsQV4q_ znosY$$a4jsc{9^!o)?+E6KUi5!1x+*Yd7P+NW&hQYT;NbVEi1@>*SFY##w%OmAL9* zdX$a*%J>`7@&$R+H^z4|eTh7FmGM8sRhGC4neHV|;sE2^zlS{PDS6~A()Kg*$OLiu zk$CbhaqB#3`vG|*d#`<+JSxtKd!a59x8ph(&h}loL!A&eoW4$RmfD=LhLTeD-jS{F_SLM9*I)M<(I` literal 0 HcmV?d00001 From 692d05b97b0637cd15df4b5bba541a053ef345d3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 10 Mar 2026 17:35:51 -0700 Subject: [PATCH 03/25] Make Log::init() fallible (#1076) Co-authored-by: Claude Opus 4.6 --- rs/hang/examples/subscribe.rs | 2 +- rs/hang/examples/video.rs | 2 +- rs/libmoq/src/api.rs | 3 ++- rs/libmoq/src/error.rs | 5 +++++ rs/moq-cli/src/main.rs | 2 +- rs/moq-clock/src/main.rs | 2 +- rs/moq-native/examples/chat.rs | 2 +- rs/moq-native/src/log.rs | 8 +++++--- rs/moq-relay/src/config.rs | 2 +- 9 files changed, 18 insertions(+), 10 deletions(-) diff --git a/rs/hang/examples/subscribe.rs b/rs/hang/examples/subscribe.rs index f624eb9ab..f48c079a3 100644 --- a/rs/hang/examples/subscribe.rs +++ b/rs/hang/examples/subscribe.rs @@ -5,7 +5,7 @@ use std::time::Duration; #[tokio::main] async fn main() -> anyhow::Result<()> { // Optional: Use moq_native to configure a logger. - moq_native::Log::new(tracing::Level::DEBUG).init(); + moq_native::Log::new(tracing::Level::DEBUG).init()?; // Create an origin that the session can publish incoming broadcasts to. let origin = moq_lite::Origin::produce(); diff --git a/rs/hang/examples/video.rs b/rs/hang/examples/video.rs index 1dd4dc5dd..052bb337e 100644 --- a/rs/hang/examples/video.rs +++ b/rs/hang/examples/video.rs @@ -4,7 +4,7 @@ use bytes::Bytes; #[tokio::main] async fn main() -> anyhow::Result<()> { // Optional: Use moq_native to configure a logger. - moq_native::Log::new(tracing::Level::DEBUG).init(); + moq_native::Log::new(tracing::Level::DEBUG).init()?; // Create an origin that we can publish to and the session can consume from. let origin = moq_lite::Origin::produce(); diff --git a/rs/libmoq/src/api.rs b/rs/libmoq/src/api.rs index 87e402bd2..50688491c 100644 --- a/rs/libmoq/src/api.rs +++ b/rs/libmoq/src/api.rs @@ -97,7 +97,8 @@ pub unsafe extern "C" fn moq_log_level(level: *const c_char, level_len: usize) - "" => moq_native::Log::default(), level => moq_native::Log::new(Level::from_str(level)?), } - .init(); + .init() + .map_err(|_| crate::error::Error::LogInit)?; Ok(()) }) diff --git a/rs/libmoq/src/error.rs b/rs/libmoq/src/error.rs index 57659c832..17eddda15 100644 --- a/rs/libmoq/src/error.rs +++ b/rs/libmoq/src/error.rs @@ -99,6 +99,10 @@ pub enum Error { #[error("level error: {0}")] Level(Arc), + /// Log initialization failed. + #[error("log init failed")] + LogInit, + /// Invalid error code conversion. #[error("invalid code")] InvalidCode, @@ -153,6 +157,7 @@ impl ffi::ReturnCode for Error { Error::Hang(_) => -18, Error::NoIndex => -19, Error::NulError(_) => -20, + Error::LogInit => -29, Error::SessionNotFound => -21, Error::OriginNotFound => -22, Error::AnnouncementNotFound => -23, diff --git a/rs/moq-cli/src/main.rs b/rs/moq-cli/src/main.rs index e26a177b5..e91ba34e7 100644 --- a/rs/moq-cli/src/main.rs +++ b/rs/moq-cli/src/main.rs @@ -81,7 +81,7 @@ async fn main() -> anyhow::Result<()> { .expect("failed to install default crypto provider"); let cli = Cli::parse(); - cli.log.init(); + cli.log.init()?; let publish = Publish::new(match &cli.command { Command::Serve { format, .. } => format, diff --git a/rs/moq-clock/src/main.rs b/rs/moq-clock/src/main.rs index 37e8ec942..df2d1825a 100644 --- a/rs/moq-clock/src/main.rs +++ b/rs/moq-clock/src/main.rs @@ -47,7 +47,7 @@ pub enum Command { #[tokio::main] async fn main() -> anyhow::Result<()> { let config = Config::parse(); - config.log.init(); + config.log.init()?; let client = config.client.init()?; diff --git a/rs/moq-native/examples/chat.rs b/rs/moq-native/examples/chat.rs index 66e2480c3..81aaae18a 100644 --- a/rs/moq-native/examples/chat.rs +++ b/rs/moq-native/examples/chat.rs @@ -3,7 +3,7 @@ #[tokio::main] async fn main() -> anyhow::Result<()> { // Optional: Use moq_native to configure a logger. - moq_native::Log::new(tracing::Level::DEBUG).init(); + moq_native::Log::new(tracing::Level::DEBUG).init()?; // Create an origin that we can publish to and the session can consume from. let origin = moq_lite::Origin::produce(); diff --git a/rs/moq-native/src/log.rs b/rs/moq-native/src/log.rs index c5e9db02e..a2164e951 100644 --- a/rs/moq-native/src/log.rs +++ b/rs/moq-native/src/log.rs @@ -34,7 +34,7 @@ impl Log { LevelFilter::from_level(self.level) } - pub fn init(&self) { + pub fn init(&self) -> anyhow::Result<()> { let filter = EnvFilter::builder() .with_default_directive(self.level().into()) // Default to our -q/-v args .from_env_lossy() // Allow overriding with RUST_LOG @@ -57,17 +57,19 @@ impl Log { tracing_subscriber::registry() .with(fmt_layer) .with(console_layer) - .init(); + .try_init()?; } #[cfg(not(feature = "tokio-console"))] { - tracing_subscriber::registry().with(fmt_layer).init(); + tracing_subscriber::registry().with(fmt_layer).try_init()?; } // Start deadlock detection thread (only in debug builds) #[cfg(debug_assertions)] std::thread::spawn(Self::deadlock_detector); + + Ok(()) } #[cfg(debug_assertions)] diff --git a/rs/moq-relay/src/config.rs b/rs/moq-relay/src/config.rs index 6c62b1f15..feac1b311 100644 --- a/rs/moq-relay/src/config.rs +++ b/rs/moq-relay/src/config.rs @@ -58,7 +58,7 @@ impl Config { config.update_from(std::env::args()); } - config.log.init(); + config.log.init()?; tracing::trace!(?config, "final config"); Ok(config) From 913138230b1148f0bec9f29b3f31e06815003060 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Mar 2026 13:58:53 -0700 Subject: [PATCH 04/25] Fix unused Result warning in moq-ffi log init (#1098) Co-authored-by: Claude Opus 4.6 (1M context) --- rs/moq-ffi/src/log.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/moq-ffi/src/log.rs b/rs/moq-ffi/src/log.rs index 65886a7df..fd9923fe8 100644 --- a/rs/moq-ffi/src/log.rs +++ b/rs/moq-ffi/src/log.rs @@ -21,7 +21,7 @@ pub fn moq_log_level(level: String) -> Result<(), MoqError> { return Err(MoqError::Log("logging already initialized".into())); } - log.init(); + log.init().map_err(|e| MoqError::Log(e.to_string()))?; Ok(()) } From 80d56306cfeb657996b7a9b71c1575a0b3387155 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Mar 2026 14:20:45 -0700 Subject: [PATCH 05/25] Add hops-based broadcast routing (#1082) Co-authored-by: Claude Opus 4.6 --- CLAUDE.md | 1 + cdn/input.tf | 22 +- cdn/main.tf | 1 + cdn/pub/demo-bbb.service.tftpl | 2 +- cdn/pub/main.tf | 4 +- cdn/pub/variables.tf | 5 + cdn/relay/dns.tf | 3 +- cdn/relay/justfile | 9 +- cdn/relay/main.tf | 9 +- cdn/relay/moq-relay.service.tftpl | 4 +- cdn/relay/variables.tf | 5 +- dev/leaf0.toml | 28 +- dev/leaf1.toml | 28 +- rs/hang/examples/video.rs | 2 +- rs/libmoq/src/publish.rs | 2 +- rs/moq-cli/src/publish.rs | 2 +- rs/moq-clock/src/main.rs | 2 +- rs/moq-ffi/src/producer.rs | 2 +- rs/moq-lite/src/ietf/subscriber.rs | 3 +- rs/moq-lite/src/lite/publisher.rs | 6 +- rs/moq-lite/src/lite/subscriber.rs | 12 +- rs/moq-lite/src/model/broadcast.rs | 68 ++++- rs/moq-lite/src/model/origin.rs | 460 +++++++++++++++++++++++------ rs/moq-mux/src/import/hls.rs | 2 +- rs/moq-mux/src/import/test/mod.rs | 2 +- rs/moq-native/examples/chat.rs | 2 +- rs/moq-relay/Cargo.toml | 2 +- rs/moq-relay/src/auth.rs | 38 +-- rs/moq-relay/src/cluster.rs | 262 ++-------------- rs/moq-relay/src/connection.rs | 8 +- rs/moq-relay/src/main.rs | 6 +- rs/moq-relay/src/web.rs | 9 +- rs/moq-relay/src/websocket.rs | 13 +- rs/moq-token-cli/src/bin.rs | 8 +- rs/moq-token/src/claims.rs | 9 +- rs/moq-token/src/key.rs | 1 + rs/moq-token/src/set.rs | 1 + 37 files changed, 548 insertions(+), 495 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index fc0fe519f..746293c3c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -102,6 +102,7 @@ match version { - Run `just fix` to automatically fix formating and easy things. - Rust tests are integrated within source files - Async tests that sleep should call `tokio::time::pause()` at the start to simulate time instantly +- Use `tokio::time::sleep()` (not `advance()`) to move time forward in paused-time tests — `sleep` both advances the clock and yields to the runtime so spawned tasks can run ## Branching Strategy diff --git a/cdn/input.tf b/cdn/input.tf index 64187d887..51b7e7aee 100644 --- a/cdn/input.tf +++ b/cdn/input.tf @@ -36,17 +36,25 @@ variable "webhook" { # instance types: https://api.linode.com/v4/linode/types locals { relays = { - usc = { - region = "us-central" # Dallas, TX - type = "g6-standard-2" # 4GB RAM, 2 vCPU, $24/mo, 4TB out + usw = { + region = "us-west" # Fremont, CA + type = "g6-standard-2" # 4GB RAM, 2 vCPU, $24/mo, 4TB out + connect = ["use"] + } + use = { + region = "us-east" # Newark, NJ + type = "g6-standard-2" + connect = ["usw", "euc"] } euc = { - region = "eu-central" # Frankfurt, Germany - type = "g6-standard-2" + region = "eu-central" # Frankfurt, Germany + type = "g6-standard-2" + connect = ["use"] } sea = { - region = "ap-south" # Singapore - type = "g6-standard-2" + region = "ap-south" # Singapore + type = "g6-standard-2" + connect = ["use"] } } } diff --git a/cdn/main.tf b/cdn/main.tf index 3c2610e5e..92aab9e43 100644 --- a/cdn/main.tf +++ b/cdn/main.tf @@ -67,6 +67,7 @@ module "relay" { module "pub" { source = "./pub" domain = var.domain + relay = "use.${var.domain}" ssh_keys = var.ssh_keys stackscript_id = linode_stackscript.bootstrap.id gcp_account_key = google_service_account_key.relay.private_key diff --git a/cdn/pub/demo-bbb.service.tftpl b/cdn/pub/demo-bbb.service.tftpl index fbfec0a6b..a42240730 100644 --- a/cdn/pub/demo-bbb.service.tftpl +++ b/cdn/pub/demo-bbb.service.tftpl @@ -23,7 +23,7 @@ ExecStart=/bin/bash -c '\ -movflags cmaf+separate_moof+delay_moov+skip_trailer+frag_every_frame \ - | \ /var/lib/moq/pkg/bin/moq-cli publish \ - --url "https://${domain}/demo?jwt=$(cat /var/lib/moq/demo-pub.jwt)" \ + --url "https://${relay}/demo?jwt=$(cat /var/lib/moq/demo-pub.jwt)" \ --name bbb \ fmp4' diff --git a/cdn/pub/main.tf b/cdn/pub/main.tf index 860f80240..23c9207ba 100644 --- a/cdn/pub/main.tf +++ b/cdn/pub/main.tf @@ -1,7 +1,7 @@ # Generate systemd service files from templates resource "local_file" "demo_bbb_service" { content = templatefile("${path.module}/demo-bbb.service.tftpl", { - domain = var.domain + relay = var.relay }) filename = "${path.module}/gen/demo-bbb.service" } @@ -9,7 +9,7 @@ resource "local_file" "demo_bbb_service" { # Publisher instance resource "linode_instance" "publisher" { label = "publisher-moq" - region = "us-central" # Dallas, TX + region = "us-east" # Newark, NJ (colocated with use relay) type = "g6-nanode-1" # Use Debian 12 as base, will be converted to NixOS via bootstrap diff --git a/cdn/pub/variables.tf b/cdn/pub/variables.tf index 05b348484..b5a5ace33 100644 --- a/cdn/pub/variables.tf +++ b/cdn/pub/variables.tf @@ -3,6 +3,11 @@ variable "domain" { type = string } +variable "relay" { + description = "Relay hostname to publish to (e.g. use.example.com)" + type = string +} + variable "ssh_keys" { description = "SSH public keys for root access" type = list(string) diff --git a/cdn/relay/dns.tf b/cdn/relay/dns.tf index 4056c3e05..cded1c990 100644 --- a/cdn/relay/dns.tf +++ b/cdn/relay/dns.tf @@ -32,7 +32,8 @@ resource "google_dns_record_set" "relay_global" { # GCP uses region codes like "us-east1", "us-west1", "europe-west3", "asia-southeast1" locals { relay_gcp_regions = { - usc = "us-central1" # Dallas, TX -> closest GCP region + usw = "us-west1" # Fremont, CA -> closest GCP region + use = "us-east1" # Newark, NJ -> closest GCP region euc = "europe-west3" # Frankfurt -> closest GCP region sea = "asia-southeast1" # Singapore -> closest GCP region } diff --git a/cdn/relay/justfile b/cdn/relay/justfile index a8f8e5f71..163cdf1e8 100644 --- a/cdn/relay/justfile +++ b/cdn/relay/justfile @@ -29,13 +29,14 @@ host node: # List the available nodes. nodes: - @echo "usc: Dallas, TX" + @echo "usw: Fremont, CA" + @echo "use: Newark, NJ" @echo "euc: Frankfurt, Germany" @echo "sea: Singapore" # Deploy moq-relay and moq-cert to all nodes [parallel] -deploy-all: (deploy "usc") (deploy "euc") (deploy "sea") +deploy-all: (deploy "usw") (deploy "use") (deploy "euc") (deploy "sea") # Deploy moq-relay and moq-cert to a specific node deploy node: @@ -53,7 +54,7 @@ deploy node: echo " Copying systemd units to /etc/systemd/system/..." rsync -az certbot-renew.service certbot-renew.timer root@$HOST:/etc/systemd/system/ - rsync -az --exclude='.gitignore' gen/ root@$HOST:/etc/systemd/system/ + rsync -az gen/moq-cert.service gen/{{node}}/ root@$HOST:/etc/systemd/system/ rsync -az ../common/vacuum.service ../common/vacuum.timer ../common/memory-alert.sh root@$HOST:/etc/systemd/system/ rsync -az --exclude='.gitignore' ../common/gen/ root@$HOST:/etc/systemd/system/ @@ -81,7 +82,7 @@ status node: echo "=== $HOST ===" ssh root@$HOST "systemctl status moq-relay --no-pager" -status-all: (status "usc") (status "euc") (status "sea") +status-all: (status "usw") (status "use") (status "euc") (status "sea") ssh node: #!/usr/bin/env bash diff --git a/cdn/relay/main.tf b/cdn/relay/main.tf index 05ff209cf..1a5f8c259 100644 --- a/cdn/relay/main.tf +++ b/cdn/relay/main.tf @@ -1,9 +1,12 @@ -# Generate systemd service files from templates +# Generate per-node systemd service files from templates resource "local_file" "moq_relay_service" { + for_each = var.relays + content = templatefile("${path.module}/moq-relay.service.tftpl", { - domain = var.domain + domain = var.domain + connect = each.value.connect }) - filename = "${path.module}/gen/moq-relay.service" + filename = "${path.module}/gen/${each.key}/moq-relay.service" } resource "local_file" "moq_cert_service" { diff --git a/cdn/relay/moq-relay.service.tftpl b/cdn/relay/moq-relay.service.tftpl index 8fe247453..d934d05e4 100644 --- a/cdn/relay/moq-relay.service.tftpl +++ b/cdn/relay/moq-relay.service.tftpl @@ -20,7 +20,9 @@ ExecStart=/var/lib/moq/pkg/bin/moq-relay \ --web-https-listen [::]:443 \ --web-https-cert /etc/letsencrypt/live/${domain}/fullchain.pem \ --web-https-key /etc/letsencrypt/live/${domain}/privkey.pem \ - --cluster-root usc.${domain} \ +%{ for peer in connect ~} + --cluster-connect ${peer}.${domain} \ +%{ endfor ~} --cluster-node %H \ --cluster-token /var/lib/moq/cluster.jwt diff --git a/cdn/relay/variables.tf b/cdn/relay/variables.tf index 7be566435..e69728a97 100644 --- a/cdn/relay/variables.tf +++ b/cdn/relay/variables.tf @@ -16,8 +16,9 @@ variable "ssh_keys" { variable "relays" { description = "Map of relay node configurations" type = map(object({ - region = string - type = string + region = string + type = string + connect = list(string) })) } diff --git a/dev/leaf0.toml b/dev/leaf0.toml index f623d71c1..c6da0a073 100644 --- a/dev/leaf0.toml +++ b/dev/leaf0.toml @@ -16,35 +16,13 @@ tls.generate = ["localhost"] # Listen for HTTP and WebSocket (TCP) connections on the given address. listen = "[::]:4444" -# This clustering scheme is very very simple for now. -# -# There is a root node that is used to connect leaf nodes together. -# Announcements flow from leaf -> root -> leaf, but any subscriptions are leaf -> leaf. -# The root node can serve (user) subscriptions too. -# -# The root node is either missing the "root" field below or it's identifical to the "node" field. -# This node acts a server only, accepting incoming connections from leaf nodes and users alike. -# -# There can be any number of leaf nodes. -# These nodes will connect to the specified root address and announce themselves via MoQ as a "broadcast". -# All nodes will discover these broadcasts and connect to other nodes. -# -# This forms an NxN mesh of nodes. -# Broadcasts are announced between all nodes with no collision detection, so duplicates are possible. -# Subscriptions will be relayed from leaf to leaf, so at most you can have: -# user -> leaf -> leaf -> user +# Clustering: connect to the root node. +# In this example, leaf nodes don't connect directly to each other; traffic gets proxied through the root. +# `connect` can be an array to connect to multiple peers, creating a tiered CDN. [cluster] -# Connect to this hostname in order to discover other nodes. connect = "localhost:4443" - -# Use the token in this file when connecting to other nodes. -# `just auth-token` will populate this file. token = "dev/root.jwt" -# My hostname, which must be accessible from other nodes. -node = "localhost:4444" - -# Each leaf node will connect to the root node and other nodes, using this configuration. [client] # QUIC uses TLS to have the client verify the server's identity. # However if you're not worried about man-in-the-middle attacks, you can disable verification: diff --git a/dev/leaf1.toml b/dev/leaf1.toml index b79e81b5f..ade763663 100644 --- a/dev/leaf1.toml +++ b/dev/leaf1.toml @@ -16,35 +16,13 @@ tls.generate = ["localhost"] # Listen for HTTP and WebSocket (TCP) connections on the given address. listen = "[::]:4445" -# This clustering scheme is very very simple for now. -# -# There is a root node that is used to connect leaf nodes together. -# Announcements flow from leaf -> root -> leaf, but any subscriptions are leaf -> leaf. -# The root node can serve (user) subscriptions too. -# -# The root node is either missing the "root" field below or it's identical to the "node" field. -# This node acts a server only, accepting incoming connections from leaf nodes and users alike. -# -# There can be any number of leaf nodes. -# These nodes will connect to the specified root address and announce themselves via MoQ as a "broadcast". -# All nodes will discover these broadcasts and connect to other nodes. -# -# This forms an NxN mesh of nodes. -# Broadcasts are announced between all nodes with no collision detection, so duplicates are possible. -# Subscriptions will be relayed from leaf to leaf, so at most you can have: -# user -> leaf -> leaf -> user +# Clustering: connect to the root node. +# In this example, leaf nodes don't connect directly to each other; traffic gets proxied through the root. +# `connect` can be an array to connect to multiple peers, creating a tiered CDN. [cluster] -# Connect to this hostname in order to discover other nodes. connect = "localhost:4443" - -# Use the token in this file when connecting to other nodes. -# `just auth-token` will populate this file. token = "dev/root.jwt" -# My hostname, which must be accessible from other nodes. -node = "localhost:4445" - -# Each leaf node will connect to the root node and other nodes, using this configuration. [client] # QUIC uses TLS to have the client verify the server's identity. # However if you're not worried about man-in-the-middle attacks, you can disable verification: diff --git a/rs/hang/examples/video.rs b/rs/hang/examples/video.rs index 052bb337e..b25c87135 100644 --- a/rs/hang/examples/video.rs +++ b/rs/hang/examples/video.rs @@ -101,7 +101,7 @@ fn create_track(broadcast: &mut moq_lite::BroadcastProducer) -> anyhow::Result anyhow::Result<()> { // Create and publish a broadcast to the origin. - let mut broadcast = moq_lite::Broadcast::produce(); + let mut broadcast = moq_lite::Broadcast::new().produce(); let track = create_track(&mut broadcast)?; // NOTE: The path is empty because we're using the URL to scope the broadcast. diff --git a/rs/libmoq/src/publish.rs b/rs/libmoq/src/publish.rs index c85bbebf0..42ed94c3c 100644 --- a/rs/libmoq/src/publish.rs +++ b/rs/libmoq/src/publish.rs @@ -16,7 +16,7 @@ pub struct Publish { impl Publish { pub fn create(&mut self) -> Result { - let mut broadcast = moq_lite::BroadcastProducer::new(); + let mut broadcast = moq_lite::Broadcast::new().produce(); let catalog = moq_mux::CatalogProducer::new(&mut broadcast)?; let id = self.broadcasts.insert((broadcast, catalog))?; diff --git a/rs/moq-cli/src/publish.rs b/rs/moq-cli/src/publish.rs index 9d13eadbc..3580eb558 100644 --- a/rs/moq-cli/src/publish.rs +++ b/rs/moq-cli/src/publish.rs @@ -35,7 +35,7 @@ pub struct Publish { impl Publish { pub fn new(format: &PublishFormat) -> anyhow::Result { - let mut broadcast = moq_lite::BroadcastProducer::default(); + let mut broadcast = moq_lite::Broadcast::new().produce(); let catalog = moq_mux::CatalogProducer::new(&mut broadcast)?; let decoder = match format { diff --git a/rs/moq-clock/src/main.rs b/rs/moq-clock/src/main.rs index df2d1825a..1ebb5f21c 100644 --- a/rs/moq-clock/src/main.rs +++ b/rs/moq-clock/src/main.rs @@ -62,7 +62,7 @@ async fn main() -> anyhow::Result<()> { match config.role { Command::Publish => { - let mut broadcast = moq_lite::Broadcast::produce(); + let mut broadcast = moq_lite::Broadcast::new().produce(); let track = broadcast.create_track(track)?; let clock = clock::Publisher::new(track); diff --git a/rs/moq-ffi/src/producer.rs b/rs/moq-ffi/src/producer.rs index 8b2edecab..29470fad7 100644 --- a/rs/moq-ffi/src/producer.rs +++ b/rs/moq-ffi/src/producer.rs @@ -38,7 +38,7 @@ impl MoqBroadcastProducer { #[uniffi::constructor] pub fn new() -> Result, MoqError> { let _guard = crate::ffi::RUNTIME.enter(); - let mut broadcast = moq_lite::BroadcastProducer::new(); + let mut broadcast = moq_lite::Broadcast::new().produce(); let catalog = moq_mux::CatalogProducer::new(&mut broadcast)?; Ok(Arc::new(Self { state: std::sync::Mutex::new(Some(BroadcastProducer { broadcast, catalog })), diff --git a/rs/moq-lite/src/ietf/subscriber.rs b/rs/moq-lite/src/ietf/subscriber.rs index 21ace2828..93dceec4b 100644 --- a/rs/moq-lite/src/ietf/subscriber.rs +++ b/rs/moq-lite/src/ietf/subscriber.rs @@ -330,7 +330,8 @@ impl Subscriber { return Ok(entry.get().producer.clone()); } Entry::Vacant(entry) => { - let broadcast = Broadcast::produce(); + // IETF protocol doesn't have hops; use 1 (remote source). + let broadcast = Broadcast::new().with_hops(1).produce(); origin.publish_broadcast(path.clone(), broadcast.consume()); entry.insert(BroadcastState { producer: broadcast.clone(), diff --git a/rs/moq-lite/src/lite/publisher.rs b/rs/moq-lite/src/lite/publisher.rs index 02b5e5b52..53beb4688 100644 --- a/rs/moq-lite/src/lite/publisher.rs +++ b/rs/moq-lite/src/lite/publisher.rs @@ -197,9 +197,9 @@ impl Publisher { Some((path, active)) => { let suffix = path.strip_prefix(&prefix).expect("origin returned invalid path").to_owned(); - if active.is_some() { - tracing::debug!(broadcast = %origin.absolute(&path), "announce"); - let msg = lite::Announce::Active { suffix, hops: 0 }; + if let Some(broadcast) = active { + tracing::debug!(broadcast = %origin.absolute(&path), hops = broadcast.info.hops, "announce"); + let msg = lite::Announce::Active { suffix, hops: broadcast.info.hops }; stream.writer.encode(&msg).await?; } else { tracing::debug!(broadcast = %origin.absolute(&path), "unannounce"); diff --git a/rs/moq-lite/src/lite/subscriber.rs b/rs/moq-lite/src/lite/subscriber.rs index 29fe365f9..73534b1f3 100644 --- a/rs/moq-lite/src/lite/subscriber.rs +++ b/rs/moq-lite/src/lite/subscriber.rs @@ -94,7 +94,8 @@ impl Subscriber { Version::Lite01 | Version::Lite02 => { let msg: lite::AnnounceInit = stream.reader.decode().await?; for path in msg.suffixes { - self.start_announce(path, &mut producers)?; + // Lite01/02 don't have hops on the wire; use 1 (remote source, unknown distance). + self.start_announce(path, Broadcast::new().with_hops(1), &mut producers)?; } } Version::Lite03 => { @@ -104,8 +105,8 @@ impl Subscriber { while let Some(announce) = stream.reader.decode_maybe::().await? { match announce { - lite::Announce::Active { suffix: path, .. } => { - self.start_announce(path, &mut producers)?; + lite::Announce::Active { suffix: path, hops } => { + self.start_announce(path, Broadcast::new().with_hops(hops + 1), &mut producers)?; } lite::Announce::Ended { suffix: path, .. } => { tracing::debug!(broadcast = %self.log_path(&path), "unannounced"); @@ -125,11 +126,12 @@ impl Subscriber { fn start_announce( &mut self, path: PathOwned, + info: Broadcast, producers: &mut HashMap, ) -> Result<(), Error> { - tracing::debug!(broadcast = %self.log_path(&path), "announce"); + tracing::debug!(broadcast = %self.log_path(&path), hops = info.hops, "announce"); - let broadcast = Broadcast::produce(); + let broadcast = info.produce(); // Make sure the peer doesn't double announce. match producers.entry(path.to_owned()) { diff --git a/rs/moq-lite/src/model/broadcast.rs b/rs/moq-lite/src/model/broadcast.rs index 4f8c540e6..f1d93c73c 100644 --- a/rs/moq-lite/src/model/broadcast.rs +++ b/rs/moq-lite/src/model/broadcast.rs @@ -3,6 +3,8 @@ use std::{ task::{Poll, ready}, }; +use std::ops::Deref; + use crate::{Error, TrackConsumer, TrackProducer, model::track::TrackWeak}; use super::Track; @@ -10,14 +12,25 @@ use super::Track; /// A collection of media tracks that can be published and subscribed to. /// /// Create via [`Broadcast::produce`] to obtain both [`BroadcastProducer`] and [`BroadcastConsumer`] pair. -#[derive(Clone, Default)] +#[derive(Clone, Debug, Default)] +#[non_exhaustive] pub struct Broadcast { - // NOTE: Broadcasts have no names because they're often relative. + /// The number of hops from the origin. + pub hops: u64, } impl Broadcast { - pub fn produce() -> BroadcastProducer { - BroadcastProducer::new() + pub fn new() -> Self { + Self::default() + } + + pub fn with_hops(mut self, hops: u64) -> Self { + self.hops = hops; + self + } + + pub fn produce(self) -> BroadcastProducer { + BroadcastProducer::new(self) } } @@ -50,18 +63,20 @@ fn modify(state: &conducer::Producer) -> Result, /// or handle on-demand requests via [Self::dynamic]. #[derive(Clone)] pub struct BroadcastProducer { + pub info: Broadcast, state: conducer::Producer, } impl Default for BroadcastProducer { fn default() -> Self { - Self::new() + Self::new(Broadcast::default()) } } impl BroadcastProducer { - pub fn new() -> Self { + pub fn new(info: Broadcast) -> Self { Self { + info, state: Default::default(), } } @@ -97,12 +112,13 @@ impl BroadcastProducer { /// Create a dynamic producer that handles on-demand track requests from consumers. pub fn dynamic(&self) -> BroadcastDynamic { - BroadcastDynamic::new(self.state.clone()) + BroadcastDynamic::new(self.info.clone(), self.state.clone()) } /// Create a consumer that can subscribe to tracks in this broadcast. pub fn consume(&self) -> BroadcastConsumer { BroadcastConsumer { + info: self.info.clone(), state: self.state.consume(), } } @@ -132,6 +148,14 @@ impl BroadcastProducer { } } +impl Deref for BroadcastProducer { + type Target = Broadcast; + + fn deref(&self) -> &Self::Target { + &self.info + } +} + #[cfg(test)] impl BroadcastProducer { pub fn assert_create_track(&mut self, track: &Track) -> TrackProducer { @@ -150,17 +174,18 @@ impl BroadcastProducer { /// Dropped when no longer needed; pending requests are automatically aborted. #[derive(Clone)] pub struct BroadcastDynamic { + info: Broadcast, state: conducer::Producer, } impl BroadcastDynamic { - fn new(state: conducer::Producer) -> Self { + fn new(info: Broadcast, state: conducer::Producer) -> Self { if let Ok(mut state) = state.write() { // If the broadcast is already closed, we can't handle any new requests. state.dynamic += 1; } - Self { state } + Self { info, state } } // A helper to automatically apply Dropped if the state is closed without an error. @@ -189,6 +214,7 @@ impl BroadcastDynamic { /// Create a consumer that can subscribe to tracks in this broadcast. pub fn consume(&self) -> BroadcastConsumer { BroadcastConsumer { + info: self.info.clone(), state: self.state.consume(), } } @@ -255,9 +281,18 @@ impl BroadcastDynamic { /// Subscribe to arbitrary broadcast/tracks. #[derive(Clone)] pub struct BroadcastConsumer { + pub info: Broadcast, state: conducer::Consumer, } +impl Deref for BroadcastConsumer { + type Target = Broadcast; + + fn deref(&self) -> &Self::Target { + &self.info + } +} + impl BroadcastConsumer { pub fn subscribe_track(&self, track: &Track) -> Result { // Upgrade to a temporary producer so we can modify the state. @@ -343,7 +378,7 @@ mod test { #[tokio::test] async fn insert() { - let mut producer = BroadcastProducer::new(); + let mut producer = Broadcast::new().produce(); let mut track1 = Track::new("track1").produce(); // Make sure we can insert before a consumer is created. @@ -369,7 +404,7 @@ mod test { #[tokio::test] async fn closed() { - let mut producer = BroadcastProducer::new(); + let mut producer = Broadcast::new().produce(); let _dynamic = producer.dynamic(); let consumer = producer.consume(); @@ -395,7 +430,7 @@ mod test { #[tokio::test] async fn requests() { - let mut producer = BroadcastProducer::new().dynamic(); + let mut producer = Broadcast::new().produce().dynamic(); let consumer = producer.consume(); let consumer2 = consumer.clone(); @@ -433,7 +468,7 @@ mod test { #[tokio::test] async fn stale_producer() { - let mut broadcast = Broadcast::produce().dynamic(); + let mut broadcast = Broadcast::new().produce().dynamic(); let consumer = broadcast.consume(); // Subscribe to a track, creating a request @@ -463,7 +498,8 @@ mod test { #[tokio::test] async fn requested_unused() { - let mut broadcast = Broadcast::produce().dynamic(); + tokio::time::pause(); + let mut broadcast = Broadcast::new().produce().dynamic(); // Subscribe to a track that doesn't exist - this creates a request let consumer1 = broadcast.consume().assert_subscribe_track(&Track::new("unknown_track")); @@ -501,8 +537,8 @@ mod test { "track producer should be unused after consumer is dropped" ); - // TODO Unfortunately, we need to sleep for a little bit to detect when unused. - tokio::time::sleep(std::time::Duration::from_millis(1)).await; + // Advance paused time to let the async cleanup task run. + tokio::time::advance(std::time::Duration::from_millis(1)).await; // Now the cleanup task should have run and we can subscribe again to the unknown track. let consumer3 = broadcast.consume().subscribe_track(&Track::new("unknown_track")); diff --git a/rs/moq-lite/src/model/origin.rs b/rs/moq-lite/src/model/origin.rs index bd68a2059..ecaf6cd6f 100644 --- a/rs/moq-lite/src/model/origin.rs +++ b/rs/moq-lite/src/model/origin.rs @@ -5,9 +5,15 @@ use std::{ use tokio::sync::mpsc; use web_async::Lock; +use std::time::Duration; + use super::BroadcastConsumer; use crate::{AsPath, Broadcast, BroadcastProducer, Path, PathOwned}; +/// Delay before reannouncing a promoted backup broadcast. +/// This avoids churn when a cascade of closures propagates through the network. +const REANNOUNCE_HOLD_DOWN: Duration = Duration::from_millis(250); + static NEXT_CONSUMER_ID: AtomicU64 = AtomicU64::new(0); #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -145,11 +151,16 @@ impl OriginNode { self.entry(dir).lock().publish(&full, broadcast, &relative); } else if let Some(existing) = &mut self.broadcast { // This node is a leaf with an existing broadcast. - let old = existing.active.clone(); - existing.active = broadcast.clone(); - existing.backup.push(old); - - self.notify.lock().reannounce(full, broadcast); + if broadcast.info.hops < existing.active.info.hops { + // New broadcast has fewer hops, so it becomes active. + let old = existing.active.clone(); + existing.active = broadcast.clone(); + existing.backup.push(old); + self.notify.lock().reannounce(full, broadcast); + } else { + // Same or more hops, just add to backup. + existing.backup.push(broadcast.clone()); + } } else { // This node is a leaf with no existing broadcast. self.broadcast = Some(OriginBroadcast { @@ -196,46 +207,90 @@ impl OriginNode { } } - // Returns true if the broadcast should be unannounced. - fn remove(&mut self, full: impl AsPath, broadcast: BroadcastConsumer, relative: impl AsPath) { + /// Remove a broadcast from this node. + /// + /// Returns `Some(promoted)` if a backup was promoted to active and needs a delayed reannounce. + /// Unannounces immediately if there are no backups. + fn remove( + &mut self, + full: impl AsPath, + broadcast: BroadcastConsumer, + relative: impl AsPath, + ) -> Option { let full = full.as_path(); let relative = relative.as_path(); if let Some((dir, relative)) = relative.next_part() { let nested = self.entry(dir); let mut locked = nested.lock(); - locked.remove(&full, broadcast, &relative); + let result = locked.remove(&full, broadcast, &relative); if locked.is_empty() { drop(locked); self.nested.remove(dir); } - } else { - let entry = match &mut self.broadcast { - Some(existing) => existing, - None => return, - }; - // See if we can remove the broadcast from the backup list. - let pos = entry.backup.iter().position(|b| b.is_clone(&broadcast)); - if let Some(pos) = pos { - entry.backup.remove(pos); - // Nothing else to do - return; - } + return result; + } - // Okay so it must be the active broadcast or else we fucked up. - assert!(entry.active.is_clone(&broadcast)); + let entry = match &mut self.broadcast { + Some(existing) => existing, + None => return None, + }; - // If there's a backup broadcast, then announce it. - if let Some(active) = entry.backup.pop() { - entry.active = active; - self.notify.lock().reannounce(full, &entry.active); - } else { - // No more backups, so remove the entry. - self.broadcast = None; - self.notify.lock().unannounce(full); + // See if we can remove the broadcast from the backup list. + let pos = entry.backup.iter().position(|b| b.is_clone(&broadcast)); + if let Some(pos) = pos { + entry.backup.remove(pos); + return None; + } + + // Okay so it must be the active broadcast or else we fucked up. + assert!(entry.active.is_clone(&broadcast)); + + // If there's a backup broadcast, pick the one with fewest hops (most recent as tiebreaker). + if !entry.backup.is_empty() { + // Reverse enumerate so that ties prefer the most recently added (last in vec). + let best = entry + .backup + .iter() + .enumerate() + .rev() + .min_by_key(|(_, b)| b.info.hops) + .map(|(i, _)| i) + .unwrap(); + let active = entry.backup.swap_remove(best); + entry.active = active.clone(); + + // Don't reannounce immediately — return the promoted backup so the caller + // can schedule a delayed reannounce (hold-down timer) to avoid churn when + // a cascade of closures is propagating through the network. + Some(active) + } else { + // No more backups, unannounce immediately. + self.broadcast = None; + self.notify.lock().unannounce(full); + None + } + } + + /// Reannounce a promoted backup if it's still the active broadcast. + /// + /// Called after the hold-down delay. If a better broadcast arrived in the meantime + /// (via publish with fewer hops), the promoted one will no longer be active and + /// this is a no-op. + fn maybe_reannounce(&mut self, full: impl AsPath, relative: impl AsPath, promoted: &BroadcastConsumer) { + let full = full.as_path(); + let relative = relative.as_path(); + + if let Some((dir, relative)) = relative.next_part() { + if let Some(nested) = self.nested.get(dir) { + nested.lock().maybe_reannounce(&full, &relative, promoted); } + } else if let Some(entry) = &self.broadcast + && entry.active.is_clone(promoted) + { + self.notify.lock().reannounce(full, &entry.active); } } @@ -360,16 +415,16 @@ impl OriginProducer { /// This is a helper method when you only want to publish a broadcast to a single origin. /// Returns [None] if the broadcast is not allowed to be published. pub fn create_broadcast(&self, path: impl AsPath) -> Option { - let broadcast = Broadcast::produce(); + let broadcast = Broadcast::new().produce(); self.publish_broadcast(path, broadcast.consume()).then_some(broadcast) } /// Publish a broadcast, announcing it to all consumers. /// /// The broadcast will be unannounced when it is closed. - /// If there is already a broadcast with the same path, then it will be replaced and reannounced. - /// If the old broadcast is closed before the new one, then nothing will happen. - /// If the new broadcast is closed before the old one, then the old broadcast will be reannounced. + /// If there is already a broadcast with the same path and more hops, it will be replaced and reannounced. + /// If the old broadcast is closed before the new one, the new broadcast will be reannounced after a hold-down delay. + /// If the new broadcast is closed before the old one, then nothing will happen. /// /// Returns false if the broadcast is not allowed to be published. pub fn publish_broadcast(&self, path: impl AsPath, broadcast: BroadcastConsumer) -> bool { @@ -387,7 +442,15 @@ impl OriginProducer { web_async::spawn(async move { broadcast.closed().await; - root.lock().remove(&full, broadcast, &rest); + let promoted = root.lock().remove(&full, broadcast, &rest); + + if let Some(promoted) = promoted { + // Hold-down timer: delay the reannounce to avoid churn when a cascade + // of closures propagates through the network. If a better path arrives + // during this window, it will reannounce immediately and this becomes a no-op. + tokio::time::sleep(REANNOUNCE_HOLD_DOWN).await; + root.lock().maybe_reannounce(&full, &rest, &promoted); + } }); true @@ -618,9 +681,11 @@ mod tests { #[tokio::test] async fn test_announce() { + tokio::time::pause(); + let origin = Origin::produce(); - let broadcast1 = Broadcast::produce(); - let broadcast2 = Broadcast::produce(); + let broadcast1 = Broadcast::new().produce(); + let broadcast2 = Broadcast::new().produce(); let mut consumer1 = origin.consume(); // Make a new consumer that should get it. @@ -682,11 +747,14 @@ mod tests { #[tokio::test] async fn test_duplicate() { + tokio::time::pause(); + let origin = Origin::produce(); - let broadcast1 = Broadcast::produce(); - let broadcast2 = Broadcast::produce(); - let broadcast3 = Broadcast::produce(); + // All same hops (0), so first becomes active, rest go to backup. + let broadcast1 = Broadcast::new().produce(); + let broadcast2 = Broadcast::new().produce(); + let broadcast3 = Broadcast::new().produce(); let consumer1 = broadcast1.consume(); let consumer2 = broadcast2.consume(); @@ -699,35 +767,36 @@ mod tests { origin.publish_broadcast("test", consumer3.clone()); assert!(consumer.consume_broadcast("test").is_some()); + // Only the first publish triggers an announce (same hops = no reannounce). consumer.assert_next("test", &consumer1); - consumer.assert_next_none("test"); - consumer.assert_next("test", &consumer2); - consumer.assert_next_none("test"); - consumer.assert_next("test", &consumer3); + consumer.assert_next_wait(); - // Drop the backup, nothing should change. + // Drop a backup, nothing should change. drop(broadcast2); - // Wait for the async task to run. + // Wait for the async cleanup task to run. tokio::time::sleep(tokio::time::Duration::from_millis(1)).await; assert!(consumer.consume_broadcast("test").is_some()); consumer.assert_next_wait(); - // Drop the active, we should reannounce. - drop(broadcast3); + // Drop the active — backup is promoted but reannounce is delayed (hold-down timer). + drop(broadcast1); - // Wait for the async task to run. + // Wait for the remove task to run, but not the hold-down. tokio::time::sleep(tokio::time::Duration::from_millis(1)).await; + consumer.assert_next_wait(); + + // Advance past the hold-down timer. Now the reannounce should fire. + tokio::time::sleep(REANNOUNCE_HOLD_DOWN + tokio::time::Duration::from_millis(1)).await; assert!(consumer.consume_broadcast("test").is_some()); consumer.assert_next_none("test"); - consumer.assert_next("test", &consumer1); + consumer.assert_next("test", &consumer3); - // Drop the final broadcast, we should unannounce. - drop(broadcast1); + // Drop the final broadcast, we should unannounce immediately. + drop(broadcast3); - // Wait for the async task to run. tokio::time::sleep(tokio::time::Duration::from_millis(1)).await; assert!(consumer.consume_broadcast("test").is_none()); @@ -737,9 +806,11 @@ mod tests { #[tokio::test] async fn test_duplicate_reverse() { + tokio::time::pause(); + let origin = Origin::produce(); - let broadcast1 = Broadcast::produce(); - let broadcast2 = Broadcast::produce(); + let broadcast1 = Broadcast::new().produce(); + let broadcast2 = Broadcast::new().produce(); origin.publish_broadcast("test", broadcast1.consume()); origin.publish_broadcast("test", broadcast2.consume()); @@ -748,21 +819,219 @@ mod tests { // This is harder, dropping the new broadcast first. drop(broadcast2); - // Wait for the cleanup async task to run. tokio::time::sleep(tokio::time::Duration::from_millis(1)).await; assert!(origin.consume_broadcast("test").is_some()); drop(broadcast1); - // Wait for the cleanup async task to run. tokio::time::sleep(tokio::time::Duration::from_millis(1)).await; assert!(origin.consume_broadcast("test").is_none()); } + #[tokio::test] + async fn test_hops_ordering() { + tokio::time::pause(); + + let origin = Origin::produce(); + + // Publish a broadcast with 3 hops. + let far = Broadcast::new().with_hops(3).produce(); + let far_consumer = far.consume(); + + let mut consumer = origin.consume(); + + origin.publish_broadcast("test", far_consumer.clone()); + consumer.assert_next("test", &far_consumer); + consumer.assert_next_wait(); + + // Now publish a closer broadcast (1 hop). It should replace the active and reannounce immediately. + let close = Broadcast::new().with_hops(1).produce(); + let close_consumer = close.consume(); + + origin.publish_broadcast("test", close_consumer.clone()); + consumer.assert_next_none("test"); + consumer.assert_next("test", &close_consumer); + consumer.assert_next_wait(); + + // Publish a broadcast with more hops (5). Should go to backup silently. + let farther = Broadcast::new().with_hops(5).produce(); + let farther_consumer = farther.consume(); + + origin.publish_broadcast("test", farther_consumer.clone()); + consumer.assert_next_wait(); + + // Drop the active (1 hop). Best backup is 3 hops. Reannounce is delayed. + drop(close); + tokio::time::sleep(tokio::time::Duration::from_millis(1)).await; + consumer.assert_next_wait(); + + // After the hold-down, the 3-hop backup is reannounced. + tokio::time::sleep(REANNOUNCE_HOLD_DOWN + tokio::time::Duration::from_millis(1)).await; + consumer.assert_next_none("test"); + consumer.assert_next("test", &far_consumer); + + // Drop the 3-hop broadcast. Best backup is 5 hops. Reannounce is delayed. + drop(far); + tokio::time::sleep(tokio::time::Duration::from_millis(1)).await; + consumer.assert_next_wait(); + + tokio::time::sleep(REANNOUNCE_HOLD_DOWN + tokio::time::Duration::from_millis(1)).await; + consumer.assert_next_none("test"); + consumer.assert_next("test", &farther_consumer); + + // Drop the last one. Should unannounce immediately. + drop(farther); + tokio::time::sleep(tokio::time::Duration::from_millis(1)).await; + + consumer.assert_next_none("test"); + consumer.assert_next_wait(); + } + + #[tokio::test] + async fn test_hops_same_no_reannounce() { + let origin = Origin::produce(); + + let b1 = Broadcast::new().with_hops(2).produce(); + let b1c = b1.consume(); + + let mut consumer = origin.consume(); + + origin.publish_broadcast("test", b1c.clone()); + consumer.assert_next("test", &b1c); + + // Publish another broadcast with same hops. Should go to backup, no reannounce. + let b2 = Broadcast::new().with_hops(2).produce(); + let _b2c = b2.consume(); + + origin.publish_broadcast("test", _b2c.clone()); + consumer.assert_next_wait(); + } + + /// When the active closes and a backup is promoted, a better publish arriving + /// during the hold-down should reannounce immediately and cancel the delayed one. + #[tokio::test] + async fn test_hold_down_superseded_by_better_publish() { + tokio::time::pause(); + + let origin = Origin::produce(); + + let b1 = Broadcast::new().with_hops(1).produce(); + let b1c = b1.consume(); + let b2 = Broadcast::new().with_hops(3).produce(); + let b2c = b2.consume(); + + let mut consumer = origin.consume(); + + origin.publish_broadcast("test", b1c.clone()); + origin.publish_broadcast("test", b2c.clone()); + consumer.assert_next("test", &b1c); + consumer.assert_next_wait(); + + // Drop the active (1 hop). Backup (3 hops) is promoted, hold-down starts. + drop(b1); + tokio::time::sleep(tokio::time::Duration::from_millis(1)).await; + consumer.assert_next_wait(); // No reannounce yet. + + // During hold-down, a better broadcast arrives (0 hops). + let b3 = Broadcast::new().with_hops(0).produce(); + let b3c = b3.consume(); + origin.publish_broadcast("test", b3c.clone()); + + // The better broadcast reannounces immediately. + consumer.assert_next_none("test"); + consumer.assert_next("test", &b3c); + consumer.assert_next_wait(); + + // After the hold-down expires, nothing happens (superseded). + tokio::time::sleep(REANNOUNCE_HOLD_DOWN).await; + consumer.assert_next_wait(); + } + + /// When the active closes and the promoted backup also closes during the hold-down, + /// we should unannounce immediately. + #[tokio::test] + async fn test_hold_down_backup_also_closes() { + tokio::time::pause(); + + let origin = Origin::produce(); + + let b1 = Broadcast::new().produce(); + let b1c = b1.consume(); + let b2 = Broadcast::new().produce(); + let b2c = b2.consume(); + + let mut consumer = origin.consume(); + + origin.publish_broadcast("test", b1c.clone()); + origin.publish_broadcast("test", b2c.clone()); + consumer.assert_next("test", &b1c); + consumer.assert_next_wait(); + + // Drop the active. Backup promoted, hold-down starts. + drop(b1); + tokio::time::sleep(tokio::time::Duration::from_millis(1)).await; + consumer.assert_next_wait(); + + // Drop the promoted backup during the hold-down. + drop(b2); + tokio::time::sleep(tokio::time::Duration::from_millis(1)).await; + + // Should unannounce immediately. + consumer.assert_next_none("test"); + consumer.assert_next_wait(); + assert!(origin.consume_broadcast("test").is_none()); + + // After the hold-down expires, nothing happens. + tokio::time::sleep(REANNOUNCE_HOLD_DOWN).await; + consumer.assert_next_wait(); + } + + /// Cascading closures: active closes, backup promoted, backup closes too. + /// The hold-down prevents churn — downstream only sees the final state. + #[tokio::test] + async fn test_hold_down_cascade() { + tokio::time::pause(); + + let origin = Origin::produce(); + + let b1 = Broadcast::new().with_hops(1).produce(); + let b1c = b1.consume(); + let b2 = Broadcast::new().with_hops(2).produce(); + let b2c = b2.consume(); + let b3 = Broadcast::new().with_hops(3).produce(); + let b3c = b3.consume(); + + let mut consumer = origin.consume(); + + origin.publish_broadcast("test", b1c.clone()); + origin.publish_broadcast("test", b2c.clone()); + origin.publish_broadcast("test", b3c.clone()); + consumer.assert_next("test", &b1c); + consumer.assert_next_wait(); + + // Drop b1 (active). b2 promoted, hold-down starts. + drop(b1); + tokio::time::sleep(tokio::time::Duration::from_millis(1)).await; + consumer.assert_next_wait(); + + // Drop b2 (promoted) during hold-down. b3 promoted, new hold-down starts. + drop(b2); + tokio::time::sleep(tokio::time::Duration::from_millis(1)).await; + consumer.assert_next_wait(); + + // After the hold-down, b3 is reannounced. + tokio::time::sleep(REANNOUNCE_HOLD_DOWN + tokio::time::Duration::from_millis(1)).await; + consumer.assert_next_none("test"); + consumer.assert_next("test", &b3c); + consumer.assert_next_wait(); + } + #[tokio::test] async fn test_double_publish() { + tokio::time::pause(); + let origin = Origin::produce(); - let broadcast = Broadcast::produce(); + let broadcast = Broadcast::new().produce(); // Ensure it doesn't crash. origin.publish_broadcast("test", broadcast.consume()); @@ -772,7 +1041,6 @@ mod tests { drop(broadcast); - // Wait for the async task to run. tokio::time::sleep(tokio::time::Duration::from_millis(1)).await; assert!(origin.consume_broadcast("test").is_none()); } @@ -781,7 +1049,7 @@ mod tests { #[should_panic] async fn test_128() { let origin = Origin::produce(); - let broadcast = Broadcast::produce(); + let broadcast = Broadcast::new().produce(); let mut consumer = origin.consume(); for i in 0..256 { @@ -796,7 +1064,7 @@ mod tests { #[tokio::test] async fn test_128_fix() { let origin = Origin::produce(); - let broadcast = Broadcast::produce(); + let broadcast = Broadcast::new().produce(); let mut consumer = origin.consume(); for i in 0..256 { @@ -812,7 +1080,7 @@ mod tests { #[tokio::test] async fn test_with_root_basic() { let origin = Origin::produce(); - let broadcast = Broadcast::produce(); + let broadcast = Broadcast::new().produce(); // Create a producer with root "/foo" let foo_producer = origin.with_root("foo").expect("should create root"); @@ -833,7 +1101,7 @@ mod tests { #[tokio::test] async fn test_with_root_nested() { let origin = Origin::produce(); - let broadcast = Broadcast::produce(); + let broadcast = Broadcast::new().produce(); // Create nested roots let foo_producer = origin.with_root("foo").expect("should create foo root"); @@ -855,7 +1123,7 @@ mod tests { #[tokio::test] async fn test_publish_only_allows() { let origin = Origin::produce(); - let broadcast = Broadcast::produce(); + let broadcast = Broadcast::new().produce(); // Create a producer that can only publish to "allowed" paths let limited_producer = origin @@ -884,9 +1152,9 @@ mod tests { #[tokio::test] async fn test_consume_only_filters() { let origin = Origin::produce(); - let broadcast1 = Broadcast::produce(); - let broadcast2 = Broadcast::produce(); - let broadcast3 = Broadcast::produce(); + let broadcast1 = Broadcast::new().produce(); + let broadcast2 = Broadcast::new().produce(); + let broadcast3 = Broadcast::new().produce(); let mut consumer = origin.consume(); @@ -914,9 +1182,9 @@ mod tests { #[tokio::test] async fn test_consume_only_multiple_prefixes() { let origin = Origin::produce(); - let broadcast1 = Broadcast::produce(); - let broadcast2 = Broadcast::produce(); - let broadcast3 = Broadcast::produce(); + let broadcast1 = Broadcast::new().produce(); + let broadcast2 = Broadcast::new().produce(); + let broadcast3 = Broadcast::new().produce(); origin.publish_broadcast("foo/test", broadcast1.consume()); origin.publish_broadcast("bar/test", broadcast2.consume()); @@ -935,7 +1203,7 @@ mod tests { #[tokio::test] async fn test_with_root_and_publish_only() { let origin = Origin::produce(); - let broadcast = Broadcast::produce(); + let broadcast = Broadcast::new().produce(); // User connects to /foo root let foo_producer = origin.with_root("foo").expect("should create foo root"); @@ -968,9 +1236,9 @@ mod tests { #[tokio::test] async fn test_with_root_and_consume_only() { let origin = Origin::produce(); - let broadcast1 = Broadcast::produce(); - let broadcast2 = Broadcast::produce(); - let broadcast3 = Broadcast::produce(); + let broadcast1 = Broadcast::new().produce(); + let broadcast2 = Broadcast::new().produce(); + let broadcast3 = Broadcast::new().produce(); // Publish broadcasts origin.publish_broadcast("foo/bar/test", broadcast1.consume()); @@ -1013,7 +1281,7 @@ mod tests { #[tokio::test] async fn test_wildcard_permission() { let origin = Origin::produce(); - let broadcast = Broadcast::produce(); + let broadcast = Broadcast::new().produce(); // Producer with root access (empty string means wildcard) let root_producer = origin.clone(); @@ -1030,8 +1298,8 @@ mod tests { #[tokio::test] async fn test_consume_broadcast_with_permissions() { let origin = Origin::produce(); - let broadcast1 = Broadcast::produce(); - let broadcast2 = Broadcast::produce(); + let broadcast1 = Broadcast::new().produce(); + let broadcast2 = Broadcast::new().produce(); origin.publish_broadcast("allowed/test", broadcast1.consume()); origin.publish_broadcast("notallowed/test", broadcast2.consume()); @@ -1058,7 +1326,7 @@ mod tests { #[tokio::test] async fn test_nested_paths_with_permissions() { let origin = Origin::produce(); - let broadcast = Broadcast::produce(); + let broadcast = Broadcast::new().produce(); // Create producer limited to "a/b/c" let limited_producer = origin @@ -1079,9 +1347,9 @@ mod tests { #[tokio::test] async fn test_multiple_consumers_with_different_permissions() { let origin = Origin::produce(); - let broadcast1 = Broadcast::produce(); - let broadcast2 = Broadcast::produce(); - let broadcast3 = Broadcast::produce(); + let broadcast1 = Broadcast::new().produce(); + let broadcast2 = Broadcast::new().produce(); + let broadcast3 = Broadcast::new().produce(); // Publish to different paths origin.publish_broadcast("foo/test", broadcast1.consume()); @@ -1116,8 +1384,8 @@ mod tests { #[tokio::test] async fn test_select_with_empty_prefix() { let origin = Origin::produce(); - let broadcast1 = Broadcast::produce(); - let broadcast2 = Broadcast::produce(); + let broadcast1 = Broadcast::new().produce(); + let broadcast2 = Broadcast::new().produce(); // User with root "demo" allowed to subscribe to "worm-node" and "foobar" let demo_producer = origin.with_root("demo").expect("should create demo root"); @@ -1143,9 +1411,9 @@ mod tests { #[tokio::test] async fn test_select_narrowing_scope() { let origin = Origin::produce(); - let broadcast1 = Broadcast::produce(); - let broadcast2 = Broadcast::produce(); - let broadcast3 = Broadcast::produce(); + let broadcast1 = Broadcast::new().produce(); + let broadcast2 = Broadcast::new().produce(); + let broadcast3 = Broadcast::new().produce(); // User with root "demo" allowed to subscribe to "worm-node" and "foobar" let demo_producer = origin.with_root("demo").expect("should create demo root"); @@ -1180,9 +1448,9 @@ mod tests { #[tokio::test] async fn test_select_multiple_roots_with_empty_prefix() { let origin = Origin::produce(); - let broadcast1 = Broadcast::produce(); - let broadcast2 = Broadcast::produce(); - let broadcast3 = Broadcast::produce(); + let broadcast1 = Broadcast::new().produce(); + let broadcast2 = Broadcast::new().produce(); + let broadcast3 = Broadcast::new().produce(); // Producer with multiple allowed roots let limited_producer = origin @@ -1209,7 +1477,7 @@ mod tests { #[tokio::test] async fn test_publish_only_with_empty_prefix() { let origin = Origin::produce(); - let broadcast = Broadcast::produce(); + let broadcast = Broadcast::new().produce(); // Producer with specific allowed paths let limited_producer = origin @@ -1231,9 +1499,9 @@ mod tests { #[tokio::test] async fn test_select_narrowing_to_deeper_path() { let origin = Origin::produce(); - let broadcast1 = Broadcast::produce(); - let broadcast2 = Broadcast::produce(); - let broadcast3 = Broadcast::produce(); + let broadcast1 = Broadcast::new().produce(); + let broadcast2 = Broadcast::new().produce(); + let broadcast3 = Broadcast::new().produce(); // Producer with broad permission let limited_producer = origin @@ -1311,6 +1579,8 @@ mod tests { // Verify unannounce also doesn't panic with trailing slash #[tokio::test] async fn test_with_root_trailing_slash_unannounce() { + tokio::time::pause(); + let origin = Origin::produce(); let prefix = "some_prefix/".to_string(); @@ -1330,8 +1600,8 @@ mod tests { #[tokio::test] async fn test_select_maintains_access_with_wider_prefix() { let origin = Origin::produce(); - let broadcast1 = Broadcast::produce(); - let broadcast2 = Broadcast::produce(); + let broadcast1 = Broadcast::new().produce(); + let broadcast2 = Broadcast::new().produce(); // Setup: user with root "demo" allowed to subscribe to specific paths let demo_producer = origin.with_root("demo").expect("should create demo root"); diff --git a/rs/moq-mux/src/import/hls.rs b/rs/moq-mux/src/import/hls.rs index 8ed4e32ac..99a009059 100644 --- a/rs/moq-mux/src/import/hls.rs +++ b/rs/moq-mux/src/import/hls.rs @@ -634,7 +634,7 @@ mod tests { #[test] fn hls_ingest_starts_without_importers() { - let mut broadcast = moq_lite::Broadcast::produce(); + let mut broadcast = moq_lite::Broadcast::new().produce(); let catalog = crate::CatalogProducer::new(&mut broadcast).unwrap(); let url = "https://example.com/master.m3u8".to_string(); let cfg = HlsConfig::new(url); diff --git a/rs/moq-mux/src/import/test/mod.rs b/rs/moq-mux/src/import/test/mod.rs index 072ff6246..ddac02624 100644 --- a/rs/moq-mux/src/import/test/mod.rs +++ b/rs/moq-mux/src/import/test/mod.rs @@ -3,7 +3,7 @@ use hang::catalog::Container; use mp4_atom::{Decode, Encode}; fn run_fmp4(data: &[u8], passthrough: bool) -> hang::Catalog { - let mut broadcast = moq_lite::BroadcastProducer::new(); + let mut broadcast = moq_lite::Broadcast::new().produce(); let catalog = crate::CatalogProducer::new(&mut broadcast).unwrap(); let config = super::Fmp4Config { passthrough }; diff --git a/rs/moq-native/examples/chat.rs b/rs/moq-native/examples/chat.rs index 81aaae18a..c459d2f91 100644 --- a/rs/moq-native/examples/chat.rs +++ b/rs/moq-native/examples/chat.rs @@ -37,7 +37,7 @@ async fn run_session(origin: moq_lite::OriginConsumer) -> anyhow::Result<()> { async fn run_broadcast(origin: moq_lite::OriginProducer) -> anyhow::Result<()> { // Create and publish a broadcast to the origin.. // A broadcast is a collection of tracks, but in this example we'll only create one. - let mut broadcast = moq_lite::Broadcast::produce(); + let mut broadcast = moq_lite::Broadcast::new().produce(); // Create a track that we'll insert into the broadcast. // A track is a series of groups representing a live stream. diff --git a/rs/moq-relay/Cargo.toml b/rs/moq-relay/Cargo.toml index 118e71826..951ab9901 100644 --- a/rs/moq-relay/Cargo.toml +++ b/rs/moq-relay/Cargo.toml @@ -40,7 +40,7 @@ rustls = { version = "0.23", features = [ "aws-lc-rs", ], default-features = false } serde = { version = "1", features = ["derive"] } -serde_with = { version = "3", features = ["json", "base64"] } +serde_with = "3" thiserror = "2" tokio = { workspace = true, features = ["full"] } toml = "0.9" diff --git a/rs/moq-relay/src/auth.rs b/rs/moq-relay/src/auth.rs index a0bf9070e..545ed1ca9 100644 --- a/rs/moq-relay/src/auth.rs +++ b/rs/moq-relay/src/auth.rs @@ -10,7 +10,6 @@ use std::time::Duration; pub struct AuthParams { pub path: String, pub jwt: Option, - pub register: Option, } impl AuthParams { @@ -24,20 +23,17 @@ impl AuthParams { pub fn from_url(url: &url::Url) -> Self { let path = url.path().to_string(); let mut jwt = None; - let mut register = None; for (k, v) in url.query_pairs() { if v.is_empty() { continue; } - match k.as_ref() { - "jwt" => jwt = Some(v.into_owned()), - "register" => register = Some(v.into_owned()), - _ => {} + if k.as_ref() == "jwt" { + jwt = Some(v.into_owned()) } } - Self { path, jwt, register } + Self { path, jwt } } } @@ -54,9 +50,6 @@ pub enum AuthError { #[error("the path does not match the root")] IncorrectRoot, - - #[error("a cluster token was expected")] - ExpectedCluster, } impl From for http::StatusCode { @@ -103,8 +96,6 @@ pub struct AuthToken { pub root: PathOwned, pub subscribe: Vec, pub publish: Vec, - pub cluster: bool, - pub register: Option, } const REFRESH_ERROR_INTERVAL: Duration = Duration::from_secs(300); @@ -297,18 +288,10 @@ impl Auth { }) .collect(); - let register = match (params.register.as_deref(), claims.cluster) { - (Some(node), true) => Some(node.to_owned()), - (Some(_), false) => return Err(AuthError::ExpectedCluster), - _ => None, - }; - Ok(AuthToken { root: root.to_owned(), subscribe, publish, - cluster: claims.cluster, - register, }) } } @@ -412,7 +395,6 @@ mod tests { let result = auth.verify(&AuthParams { path: "/any/path".into(), jwt: Some("fake-token".into()), - ..Default::default() }); assert!(result.is_err()); @@ -441,7 +423,6 @@ mod tests { let token = auth.verify(&AuthParams { path: "/room/123".into(), jwt: Some(token), - ..Default::default() })?; assert_eq!(token.root, "room/123".as_path()); assert_eq!(token.subscribe, vec!["".as_path()]); @@ -472,7 +453,6 @@ mod tests { let result = auth.verify(&AuthParams { path: "/secret".into(), jwt: Some(token), - ..Default::default() }); assert!(result.is_err()); @@ -501,7 +481,6 @@ mod tests { let token = auth.verify(&AuthParams { path: "/room/123".into(), jwt: Some(token), - ..Default::default() })?; assert_eq!(token.root, "room/123".as_path()); assert_eq!(token.subscribe, vec!["bob".as_path()]); @@ -531,7 +510,6 @@ mod tests { let token = auth.verify(&AuthParams { path: "/room/123".into(), jwt: Some(token), - ..Default::default() })?; assert_eq!(token.subscribe, vec!["".as_path()]); assert_eq!(token.publish, vec![]); @@ -560,7 +538,6 @@ mod tests { let token = auth.verify(&AuthParams { path: "/room/123".into(), jwt: Some(token), - ..Default::default() })?; assert_eq!(token.subscribe, vec![]); assert_eq!(token.publish, vec!["bob".as_path()]); @@ -590,7 +567,6 @@ mod tests { let token = auth.verify(&AuthParams { path: "/room/123/alice".into(), jwt: Some(token), - ..Default::default() })?; // Root should be updated to the more specific path @@ -624,7 +600,6 @@ mod tests { let token = auth.verify(&AuthParams { path: "/room/123/alice".into(), jwt: Some(token), - ..Default::default() })?; assert_eq!(token.root, "room/123/alice".as_path()); @@ -658,7 +633,6 @@ mod tests { let token = auth.verify(&AuthParams { path: "/room/123/bob".into(), jwt: Some(token), - ..Default::default() })?; assert_eq!(token.root, "room/123/bob".as_path()); @@ -691,7 +665,6 @@ mod tests { let verified = auth.verify(&AuthParams { path: "/room/123/alice".into(), jwt: Some(token.clone()), - ..Default::default() })?; assert_eq!(verified.root, "room/123/alice".as_path()); @@ -704,7 +677,6 @@ mod tests { let verified = auth.verify(&AuthParams { path: "/room/123/bob".into(), jwt: Some(token), - ..Default::default() })?; assert_eq!(verified.root, "room/123/bob".as_path()); @@ -738,7 +710,6 @@ mod tests { let verified = auth.verify(&AuthParams { path: "/room/123/users".into(), jwt: Some(token.clone()), - ..Default::default() })?; assert_eq!(verified.root, "room/123/users".as_path()); @@ -750,7 +721,6 @@ mod tests { let verified = auth.verify(&AuthParams { path: "/room/123/users/alice".into(), jwt: Some(token), - ..Default::default() })?; assert_eq!(verified.root, "room/123/users/alice".as_path()); @@ -784,7 +754,6 @@ mod tests { let verified = auth.verify(&AuthParams { path: "/room/123/alice".into(), jwt: Some(token), - ..Default::default() })?; // Should remain read-only @@ -803,7 +772,6 @@ mod tests { let verified = auth.verify(&AuthParams { path: "/room/123/alice".into(), jwt: Some(token), - ..Default::default() })?; // Should remain write-only diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index 9f8933301..7fd30d211 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -1,54 +1,28 @@ -use std::{collections::HashMap, path::PathBuf}; +use std::path::PathBuf; use anyhow::Context; -use moq_lite::{Broadcast, BroadcastConsumer, BroadcastProducer, Origin, OriginConsumer, OriginProducer}; +use moq_lite::{BroadcastConsumer, Origin, OriginProducer}; use tracing::Instrument; use url::Url; -use crate::AuthToken; - #[serde_with::serde_as] #[derive(clap::Args, Clone, Debug, serde::Serialize, serde::Deserialize, Default)] #[serde_with::skip_serializing_none] #[serde(default, deny_unknown_fields)] pub struct ClusterConfig { - /// Connect to this hostname in order to discover other nodes. - #[serde(alias = "connect")] + /// Connect to these hostnames to form the cluster. #[arg( - id = "cluster-root", - long = "cluster-root", - env = "MOQ_CLUSTER_ROOT", - alias = "cluster-connect" + id = "cluster-connect", + long = "cluster-connect", + env = "MOQ_CLUSTER_CONNECT", + value_delimiter = ',' )] - pub root: Option, + #[serde_as(as = "serde_with::OneOrMany<_>")] + pub connect: Vec, /// Use the token in this file when connecting to other nodes. #[arg(id = "cluster-token", long = "cluster-token", env = "MOQ_CLUSTER_TOKEN")] pub token: Option, - - /// Our hostname which we advertise to other nodes. - /// - // TODO Remove alias once we've migrated to the new name. - #[serde(alias = "advertise")] - #[arg( - id = "cluster-node", - long = "cluster-node", - env = "MOQ_CLUSTER_NODE", - alias = "cluster-advertise" - )] - pub node: Option, - - /// The prefix to use for cluster announcements. - /// Defaults to "internal/origins". - /// - /// WARNING: This should not be accessible by users unless authentication is disabled (YOLO). - #[arg( - id = "cluster-prefix", - long = "cluster-prefix", - default_value = "internal/origins", - env = "MOQ_CLUSTER_PREFIX" - )] - pub prefix: String, } #[derive(Clone)] @@ -56,14 +30,9 @@ pub struct Cluster { config: ClusterConfig, client: moq_native::Client, - // Broadcasts announced by local clients (users). - pub primary: OriginProducer, - - // Broadcasts announced by remote servers (cluster). - pub secondary: OriginProducer, - - // Broadcasts announced by local clients and remote servers. - pub combined: OriginProducer, + // All broadcasts, both local and remote. + // Hops-based routing ensures the shortest path is preferred. + pub origin: OriginProducer, } impl Cluster { @@ -71,82 +40,20 @@ impl Cluster { Cluster { config, client, - primary: Origin::produce(), - secondary: Origin::produce(), - combined: Origin::produce(), + origin: Origin::produce(), } } - // For a given auth token, return the origin that should be used for the session. - pub fn subscriber(&self, token: &AuthToken) -> Option { - // These broadcasts will be served to the session (when it subscribes). - // If this is a cluster node, then only publish our primary broadcasts. - // Otherwise publish everything. - let subscribe_origin = match token.cluster { - true => &self.primary, - false => &self.combined, - }; - - // Scope the origin to our root. - let subscribe_origin = subscribe_origin.with_root(&token.root)?; - subscribe_origin.consume_only(&token.subscribe) - } - - // For a given auth token, return the origin that should be used for the session. - pub fn publisher(&self, token: &AuthToken) -> Option { - // If this is a cluster node, then add its broadcasts to the secondary origin. - // That way we won't publish them to other cluster nodes. - let publish_origin = match token.cluster { - true => &self.secondary, - false => &self.primary, - }; - - let publish_origin = publish_origin.with_root(&token.root)?; - publish_origin.publish_only(&token.publish) - } - - // Register a cluster node's presence. - // - // Returns a [ClusterRegistration] that should be kept alive for the duration of the session. - pub fn register(&self, token: &AuthToken) -> Option { - let node = token.register.clone()?; - let broadcast = Broadcast::produce(); - - let path = moq_lite::Path::new(&self.config.prefix).join(&node); - self.primary.publish_broadcast(path, broadcast.consume()); - - Some(ClusterRegistration::new(node, broadcast)) - } - pub fn get(&self, broadcast: &str) -> Option { - self.primary - .consume_broadcast(broadcast) - .or_else(|| self.secondary.consume_broadcast(broadcast)) + self.origin.consume_broadcast(broadcast) } pub async fn run(self) -> anyhow::Result<()> { - // If we're using a root node, then we have to connect to it. - // Otherwise, we're the root node so we wait for other nodes to connect to us. - let Some(root) = self - .config - .root - .clone() - .filter(|connect| Some(connect) != self.config.node.as_ref()) - else { - tracing::info!("running as root, accepting leaf nodes"); - self.run_combined().await?; - anyhow::bail!("combined connection closed"); - }; - - // Subscribe to available origins from secondary (what we learn from other nodes). - // Use with_root to automatically strip the prefix from announced paths. - let origins = self - .secondary - .with_root(&self.config.prefix) - .context("no authorized origins")?; + if self.config.connect.is_empty() { + return Ok(()); + } // If the token is provided, read it from the disk and use it in the query parameter. - // TODO put this in an AUTH header once WebTransport supports it. let token = match &self.config.token { Some(path) => std::fs::read_to_string(path) .context("failed to read token")? @@ -155,114 +62,31 @@ impl Cluster { None => "".to_string(), }; - let local = self.config.node.clone().context("missing node")?; - - // Create a dummy broadcast that we don't close so run_remote doesn't close. - let noop = Broadcast::produce(); - - // Despite returning a Result, we should NEVER return an Ok - tokio::select! { - res = self.clone().run_remote(&root, Some(local.as_str()), token.clone(), noop.consume()) => { - res.context("failed to connect to root")?; - anyhow::bail!("connection to root closed"); - } - res = self.clone().run_remotes(origins.consume(), token) => { - res.context("failed to connect to remotes")?; - anyhow::bail!("connection to remotes closed"); - } - res = self.run_combined() => { - res.context("failed to run combined")?; - anyhow::bail!("combined connection closed"); - } - } - } - - // Shovel broadcasts from the primary and secondary origins into the combined origin. - async fn run_combined(self) -> anyhow::Result<()> { - let mut primary = self.primary.consume(); - let mut secondary = self.secondary.consume(); - - loop { - let (name, broadcast) = tokio::select! { - biased; - Some(primary) = primary.announced() => primary, - Some(secondary) = secondary.announced() => secondary, - else => return Ok(()), - }; - - if let Some(broadcast) = broadcast { - self.combined.publish_broadcast(&name, broadcast); - } - } - } - - async fn run_remotes(self, mut origins: OriginConsumer, token: String) -> anyhow::Result<()> { - // Cancel tasks when the origin is closed. - let mut active: HashMap = HashMap::new(); - - // Discover other origins. - // NOTE: The root node will connect to all other nodes as a client, ignoring the existing (server) connection. - // This ensures that nodes are advertising a valid hostname before any tracks get announced. - while let Some((node, origin)) = origins.announced().await { - if self.config.node.as_deref() == Some(node.as_str()) { - // Skip ourselves. - continue; - } - - let Some(origin) = origin else { - tracing::info!(%node, "origin cancelled"); - active.remove(node.as_str()).unwrap().abort(); - continue; - }; - - tracing::info!(%node, "discovered origin"); - + let mut tasks = tokio::task::JoinSet::new(); + for remote in self.config.connect.clone() { let this = self.clone(); let token = token.clone(); - let node2 = node.clone(); - - let handle = tokio::spawn( - async move { - match this.run_remote(node2.as_str(), None, token, origin).await { - Ok(()) => tracing::info!(%node2, "origin closed"), - Err(err) => tracing::warn!(%err, %node2, "origin error"), - } - } - .in_current_span(), - ); + tasks.spawn(async move { this.run_remote(&remote, token).await }.in_current_span()); + } - active.insert(node.to_string(), handle.abort_handle()); + while let Some(res) = tasks.join_next().await { + res??; } Ok(()) } #[tracing::instrument("remote", skip_all, err, fields(%remote))] - async fn run_remote( - mut self, - remote: &str, - register: Option<&str>, - token: String, - origin: BroadcastConsumer, - ) -> anyhow::Result<()> { + async fn run_remote(self, remote: &str, token: String) -> anyhow::Result<()> { let mut url = Url::parse(&format!("https://{remote}/"))?; - { - let mut q = url.query_pairs_mut(); - if !token.is_empty() { - q.append_pair("jwt", &token); - } - if let Some(register) = register { - q.append_pair("register", register); - } + if !token.is_empty() { + url.query_pairs_mut().append_pair("jwt", &token); } + let mut backoff = 1; loop { - let res = tokio::select! { - biased; - _ = origin.closed() => break, - res = self.run_remote_once(&url) => res, - }; + let res = self.run_remote_once(&url).await; match res { Ok(()) => backoff = 1, @@ -274,17 +98,14 @@ impl Cluster { let timeout = tokio::time::Duration::from_secs(backoff); if timeout > tokio::time::Duration::from_secs(300) { - // 5 minutes of backoff is enough, just give up. anyhow::bail!("remote connection keep failing, giving up"); } tokio::time::sleep(timeout).await; } - - Ok(()) } - async fn run_remote_once(&mut self, url: &Url) -> anyhow::Result<()> { + async fn run_remote_once(&self, url: &Url) -> anyhow::Result<()> { let mut log_url = url.clone(); log_url.set_query(None); tracing::info!(url = %log_url, "connecting to remote"); @@ -292,8 +113,8 @@ impl Cluster { let session = self .client .clone() - .with_publish(self.primary.consume()) - .with_consume(self.secondary.clone()) + .with_publish(self.origin.consume()) + .with_consume(self.origin.clone()) .connect(url.clone()) .await .context("failed to connect to remote")?; @@ -301,24 +122,3 @@ impl Cluster { session.closed().await.map_err(Into::into) } } - -pub struct ClusterRegistration { - // The name of the node. - node: String, - - // The announcement, send to other nodes. - broadcast: BroadcastProducer, -} - -impl ClusterRegistration { - pub fn new(node: String, broadcast: BroadcastProducer) -> Self { - tracing::info!(%node, "registered cluster client"); - ClusterRegistration { node, broadcast } - } -} -impl Drop for ClusterRegistration { - fn drop(&mut self) { - tracing::info!(%self.node, "unregistered cluster client"); - let _ = self.broadcast.abort(moq_lite::Error::Cancel); - } -} diff --git a/rs/moq-relay/src/connection.rs b/rs/moq-relay/src/connection.rs index dd6261cce..c4f7e0fd9 100644 --- a/rs/moq-relay/src/connection.rs +++ b/rs/moq-relay/src/connection.rs @@ -28,9 +28,9 @@ impl Connection { } }; - let publish = self.cluster.publisher(&token); - let subscribe = self.cluster.subscriber(&token); - let registration = self.cluster.register(&token); + let origin = self.cluster.origin.with_root(&token.root); + let publish = origin.as_ref().and_then(|o| o.publish_only(&token.publish)); + let subscribe = origin.as_ref().and_then(|o| o.consume_only(&token.subscribe)); let transport = self.request.transport(); match (&publish, &subscribe) { @@ -62,9 +62,7 @@ impl Connection { tracing::info!(version = %session.version(), transport, "negotiated"); // Wait until the session is closed. - // Keep registration alive so the cluster node stays announced. session.closed().await?; - drop(registration); Ok(()) } } diff --git a/rs/moq-relay/src/main.rs b/rs/moq-relay/src/main.rs index 7133d0301..6ffb9a02b 100644 --- a/rs/moq-relay/src/main.rs +++ b/rs/moq-relay/src/main.rs @@ -55,7 +55,11 @@ async fn main() -> anyhow::Result<()> { let cluster = Cluster::new(config.cluster, client); let cloned = cluster.clone(); - tokio::spawn(async move { cloned.run().await.expect("cluster failed") }); + tokio::spawn(async move { + if let Err(err) = cloned.run().await { + panic!("cluster failed: {err}"); + } + }); // Create a web server too. let web = Web::new( diff --git a/rs/moq-relay/src/web.rs b/rs/moq-relay/src/web.rs index 71a39bdce..141ead451 100644 --- a/rs/moq-relay/src/web.rs +++ b/rs/moq-relay/src/web.rs @@ -159,7 +159,6 @@ async fn serve_fingerprint(State(state): State>) -> String { #[derive(Debug, serde::Deserialize)] pub(crate) struct AuthQuery { pub(crate) jwt: Option, - pub(crate) register: Option, } #[derive(Debug, serde::Deserialize)] @@ -231,10 +230,10 @@ async fn serve_announced( let params = AuthParams { path: prefix, jwt: query.jwt, - register: query.register, }; let token = state.auth.verify(¶ms)?; - let Some(mut origin) = state.cluster.subscriber(&token) else { + let scoped = state.cluster.origin.with_root(&token.root); + let Some(mut origin) = scoped.and_then(|o| o.consume_only(&token.subscribe)) else { return Err(StatusCode::UNAUTHORIZED.into()); }; @@ -268,11 +267,11 @@ async fn serve_fetch( let auth = AuthParams { path: broadcast.clone(), jwt: params.auth.jwt, - register: params.auth.register, }; let token = state.auth.verify(&auth)?; - let Some(origin) = state.cluster.subscriber(&token) else { + let scoped = state.cluster.origin.with_root(&token.root); + let Some(origin) = scoped.and_then(|o| o.consume_only(&token.subscribe)) else { return Err(StatusCode::UNAUTHORIZED.into()); }; diff --git a/rs/moq-relay/src/websocket.rs b/rs/moq-relay/src/websocket.rs index 5017a1065..f49627f47 100644 --- a/rs/moq-relay/src/websocket.rs +++ b/rs/moq-relay/src/websocket.rs @@ -23,15 +23,11 @@ pub(crate) async fn serve_ws( ) -> axum::response::Result { let ws = ws.protocols(["webtransport"]); - let params = AuthParams { - path, - jwt: query.jwt, - register: query.register, - }; + let params = AuthParams { path, jwt: query.jwt }; let token = state.auth.verify(¶ms)?; - let publish = state.cluster.publisher(&token); - let subscribe = state.cluster.subscriber(&token); - let registration = state.cluster.register(&token); + let origin = state.cluster.origin.with_root(&token.root); + let publish = origin.as_ref().and_then(|o| o.publish_only(&token.publish)); + let subscribe = origin.as_ref().and_then(|o| o.consume_only(&token.subscribe)); if publish.is_none() && subscribe.is_none() { // Bad token, we can't publish or subscribe. @@ -52,7 +48,6 @@ pub(crate) async fn serve_ws( }) .with(tungstenite_to_axum); let _ = handle_socket(id, socket, publish, subscribe).await; - drop(registration); })) } diff --git a/rs/moq-token-cli/src/bin.rs b/rs/moq-token-cli/src/bin.rs index 128cc4e6a..079b3c20e 100644 --- a/rs/moq-token-cli/src/bin.rs +++ b/rs/moq-token-cli/src/bin.rs @@ -49,10 +49,8 @@ enum Commands { #[arg(long)] publish: Vec, - /// If true, then this client is considered a cluster node. - /// Both the client and server will only announce broadcasts from non-cluster clients. - /// This avoids convoluted routing, as only the primary origin will announce. - #[arg(long)] + /// Deprecated: Previously used to mark cluster nodes. No longer has any effect. + #[arg(long, hide = true)] cluster: bool, /// If specified, the user can subscribe to any matching path prefixes. @@ -90,6 +88,7 @@ fn main() -> anyhow::Result<()> { key.to_file(&cli.key)?; } + #[allow(deprecated)] Commands::Sign { root, publish, @@ -100,6 +99,7 @@ fn main() -> anyhow::Result<()> { } => { let key = moq_token::Key::from_file(cli.key)?; + #[allow(deprecated)] let payload = moq_token::Claims { root, publish, diff --git a/rs/moq-token/src/claims.rs b/rs/moq-token/src/claims.rs index 1a480da3c..fb7c7e936 100644 --- a/rs/moq-token/src/claims.rs +++ b/rs/moq-token/src/claims.rs @@ -42,11 +42,9 @@ pub struct Claims { )] pub publish: Vec, - /// If true, then this client is considered a cluster node. - /// Both the client and server will only announce broadcasts from non-cluster clients. - /// This avoids convoluted routing, as only the primary origin will announce. - // - // TODO This shouldn't be part of the token. + /// Deprecated: Previously used to mark cluster nodes. + /// Kept for backwards compatibility with existing tokens. + #[deprecated] #[serde(default, rename = "cluster", skip_serializing_if = "is_false")] pub cluster: bool, @@ -83,6 +81,7 @@ impl Claims { } #[cfg(test)] +#[allow(deprecated)] mod tests { use super::*; diff --git a/rs/moq-token/src/key.rs b/rs/moq-token/src/key.rs index 8d16deb9a..15e1bf00f 100644 --- a/rs/moq-token/src/key.rs +++ b/rs/moq-token/src/key.rs @@ -512,6 +512,7 @@ where } #[cfg(test)] +#[allow(deprecated)] mod tests { use super::*; use std::time::{Duration, SystemTime}; diff --git a/rs/moq-token/src/set.rs b/rs/moq-token/src/set.rs index 9b8778842..30acd288e 100644 --- a/rs/moq-token/src/set.rs +++ b/rs/moq-token/src/set.rs @@ -139,6 +139,7 @@ pub async fn load_keys(jwks_uri: &str) -> anyhow::Result { } #[cfg(test)] +#[allow(deprecated)] mod tests { use super::*; use crate::Algorithm; From 247a393ad865c7499cd03630825d2d45bf40bee5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Mar 2026 12:21:41 -0700 Subject: [PATCH 06/25] Reapply "Rename next_group to recv_group for clarity (#1135)" This reverts commit bd68fa245cb8c0ebddd9b4e1a9275d564b396b2f. --- doc/js/@moq/lite.md | 2 +- doc/rs/index.md | 2 +- js/clock/src/main.ts | 2 +- js/hang/src/container/legacy.ts | 2 +- js/lite/examples/subscribe.ts | 2 +- js/lite/src/ietf/publisher.ts | 2 +- js/lite/src/lite/publisher.ts | 2 +- js/lite/src/track.ts | 13 ++++++++++- js/watch/src/audio/decoder.ts | 2 +- js/watch/src/video/decoder.ts | 2 +- rs/hang/src/catalog/consumer.rs | 2 +- rs/hang/src/container/consumer.rs | 4 ++-- rs/moq-clock/src/clock.rs | 2 +- rs/moq-lite/src/ietf/publisher.rs | 2 +- rs/moq-lite/src/lite/publisher.rs | 2 +- rs/moq-lite/src/model/track.rs | 38 +++++++++++++++++++++++-------- rs/moq-native/tests/backend.rs | 12 +++++----- rs/moq-native/tests/broadcast.rs | 18 +++++++-------- rs/moq-relay/src/web.rs | 2 +- 19 files changed, 71 insertions(+), 42 deletions(-) diff --git a/doc/js/@moq/lite.md b/doc/js/@moq/lite.md index efb470d15..682d18978 100644 --- a/doc/js/@moq/lite.md +++ b/doc/js/@moq/lite.md @@ -81,7 +81,7 @@ if (request) { // Read data as it arrives for (;;) { - const group = await track.nextGroup(); + const group = await track.recvGroup(); if (!group) break; for (;;) { diff --git a/doc/rs/index.md b/doc/rs/index.md index 5b8f2d615..0f0ed3ee6 100644 --- a/doc/rs/index.md +++ b/doc/rs/index.md @@ -241,7 +241,7 @@ async fn main() -> Result<(), Box> { let mut track = broadcast.subscribe("chat").await?; // Read groups and frames - while let Some(group) = track.next_group().await? { + while let Some(group) = track.recv_group().await? { while let Some(frame) = group.read().await? { println!("Received: {:?}", frame); } diff --git a/js/clock/src/main.ts b/js/clock/src/main.ts index 873e764e8..651646630 100755 --- a/js/clock/src/main.ts +++ b/js/clock/src/main.ts @@ -147,7 +147,7 @@ async function subscribe(config: Config) { // Handle groups and frames like the Rust implementation for (;;) { - const group = await track.nextGroup(); + const group = await track.recvGroup(); if (!group) { console.log("❌ Connection ended"); break; diff --git a/js/hang/src/container/legacy.ts b/js/hang/src/container/legacy.ts index 70e265894..68498f88e 100644 --- a/js/hang/src/container/legacy.ts +++ b/js/hang/src/container/legacy.ts @@ -109,7 +109,7 @@ export class Consumer { async #run() { // Start fetching groups in the background for (;;) { - const consumer = await this.#track.nextGroup(); + const consumer = await this.#track.recvGroup(); if (!consumer) break; // To improve TTV, we always start with the first group. diff --git a/js/lite/examples/subscribe.ts b/js/lite/examples/subscribe.ts index e294fdd6a..d2b4ca913 100644 --- a/js/lite/examples/subscribe.ts +++ b/js/lite/examples/subscribe.ts @@ -12,7 +12,7 @@ async function main() { // Read data as it arrives for (;;) { - const group = await track.nextGroup(); + const group = await track.recvGroup(); if (!group) break; for (;;) { diff --git a/js/lite/src/ietf/publisher.ts b/js/lite/src/ietf/publisher.ts index 3bd31b6f4..216f47994 100644 --- a/js/lite/src/ietf/publisher.ts +++ b/js/lite/src/ietf/publisher.ts @@ -161,7 +161,7 @@ export class Publisher { // Serve track groups, racing with stream close (= Unsubscribe) const serving = (async () => { for (;;) { - const group = await track.nextGroup(); + const group = await track.recvGroup(); if (!group) return; void this.#runGroup(msg.requestId, group); } diff --git a/js/lite/src/lite/publisher.ts b/js/lite/src/lite/publisher.ts index e50d40a93..56a06ba53 100644 --- a/js/lite/src/lite/publisher.ts +++ b/js/lite/src/lite/publisher.ts @@ -201,7 +201,7 @@ export class Publisher { async #runTrack(sub: bigint, broadcast: Path.Valid, track: Track, stream: Writer) { try { for (;;) { - const next = track.nextGroup(); + const next = track.recvGroup(); const group = await Promise.race([next, stream.closed]); if (!group) { next.then((group) => group?.close()).catch(() => {}); diff --git a/js/lite/src/track.ts b/js/lite/src/track.ts index 16dfe17fe..0d4b0242f 100644 --- a/js/lite/src/track.ts +++ b/js/lite/src/track.ts @@ -91,7 +91,13 @@ export class Track { group.close(); } - async nextGroup(): Promise { + /** + * Receive the next group available on this track. + * + * Groups may arrive out of order or with gaps due to network conditions. + * Use `OrderedConsumer` (in `@moq/hang`) if you need groups in sequence order. + */ + async recvGroup(): Promise { for (;;) { const groups = this.state.groups.peek(); if (groups.length > 0) { @@ -106,6 +112,11 @@ export class Track { } } + /** @deprecated Use {@link recvGroup} instead. */ + async nextGroup(): Promise { + return this.recvGroup(); + } + async readFrame(): Promise { return (await this.readFrameSequence())?.data; } diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 411f29d1a..e2dc082b4 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -277,7 +277,7 @@ export class Decoder { // Process data segments // TODO: Use a consumer wrapper for CMAF to support latency control for (;;) { - const group = await sub.nextGroup(); + const group = await sub.recvGroup(); if (!group) break; effect.spawn(async () => { diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index 9d42dad3c..495c171ec 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -379,7 +379,7 @@ class DecoderTrack { // Process data segments // TODO: Use a consumer wrapper for CMAF to support latency control for (;;) { - const group = await Promise.race([sub.nextGroup(), effect.cancel]); + const group = await Promise.race([sub.recvGroup(), effect.cancel]); if (!group) break; effect.spawn(async () => { diff --git a/rs/hang/src/catalog/consumer.rs b/rs/hang/src/catalog/consumer.rs index 7db8518ab..1339249a4 100644 --- a/rs/hang/src/catalog/consumer.rs +++ b/rs/hang/src/catalog/consumer.rs @@ -24,7 +24,7 @@ impl CatalogConsumer { pub async fn next(&mut self) -> Result> { loop { tokio::select! { - res = self.track.next_group() => { + res = self.track.recv_group() => { match res? { Some(group) => { // Use the new group. diff --git a/rs/hang/src/container/consumer.rs b/rs/hang/src/container/consumer.rs index e19e2ea60..e7e18c1d4 100644 --- a/rs/hang/src/container/consumer.rs +++ b/rs/hang/src/container/consumer.rs @@ -192,7 +192,7 @@ impl OrderedConsumer { // Returns Pending until all groups have been consumed. fn poll_read_finish(&mut self, waiter: &conducer::Waiter) -> Poll> { loop { - let Some(group) = ready!(self.track.poll_next_group(waiter)?) else { + let Some(group) = ready!(self.track.poll_recv_group(waiter)?) else { // Track is finished. return Poll::Ready(Ok(())); }; @@ -861,7 +861,7 @@ mod tests { let finisher = tokio::spawn(async move { // After consumer has buffered group 1's frames via buffer_until... tokio::time::sleep(Duration::from_millis(20)).await; - // Write group 2: next_group fires, drops current buffer_until for group 1 + // Write group 2: recv_group fires, drops current buffer_until for group 1 write_group(&mut track, 2, &[ts(200_000)]); // Then finish group 0: consumer proceeds, re-creates buffer_until for group 1 tokio::time::sleep(Duration::from_millis(20)).await; diff --git a/rs/moq-clock/src/clock.rs b/rs/moq-clock/src/clock.rs index 2c6ef7f2e..527f218a6 100644 --- a/rs/moq-clock/src/clock.rs +++ b/rs/moq-clock/src/clock.rs @@ -80,7 +80,7 @@ impl Subscriber { } pub async fn run(mut self) -> anyhow::Result<()> { - while let Some(mut group) = self.track.next_group().await? { + while let Some(mut group) = self.track.recv_group().await? { let base = group .read_frame() .await diff --git a/rs/moq-lite/src/ietf/publisher.rs b/rs/moq-lite/src/ietf/publisher.rs index 438c410ae..95596b3fc 100644 --- a/rs/moq-lite/src/ietf/publisher.rs +++ b/rs/moq-lite/src/ietf/publisher.rs @@ -225,7 +225,7 @@ impl Publisher { while tasks.next().await.is_some() {} false } => unreachable!(), - Some(group) = track.next_group().transpose() => group, + Some(group) = track.recv_group().transpose() => group, else => return Ok(()), }?; diff --git a/rs/moq-lite/src/lite/publisher.rs b/rs/moq-lite/src/lite/publisher.rs index 53beb4688..57db12d7b 100644 --- a/rs/moq-lite/src/lite/publisher.rs +++ b/rs/moq-lite/src/lite/publisher.rs @@ -310,7 +310,7 @@ impl Publisher { while tasks.next().await.is_some() {} false } => unreachable!(), - Some(group) = track.next_group().transpose() => group, + Some(group) = track.recv_group().transpose() => group, else => return Ok(()), }?; diff --git a/rs/moq-lite/src/model/track.rs b/rs/moq-lite/src/model/track.rs index d5baa2634..5fd931d31 100644 --- a/rs/moq-lite/src/model/track.rs +++ b/rs/moq-lite/src/model/track.rs @@ -406,13 +406,17 @@ impl TrackConsumer { }) } - /// Poll for the next group without blocking. + /// Poll for the next group received over the network, without blocking. + /// + /// Groups may arrive out of order or with gaps due to network conditions. + /// Use `OrderedConsumer` if you need groups in sequence order, + /// skipping those that arrive too late. /// /// Returns `Poll::Ready(Ok(Some(group)))` when a group is available, /// `Poll::Ready(Ok(None))` when the track is finished, /// `Poll::Ready(Err(e))` when the track has been aborted, or /// `Poll::Pending` when no group is available yet. - pub fn poll_next_group(&mut self, waiter: &conducer::Waiter) -> Poll>> { + pub fn poll_recv_group(&mut self, waiter: &conducer::Waiter) -> Poll>> { let Some((consumer, found_index)) = ready!(self.poll(waiter, |state| state.poll_next_group(self.index, self.min_sequence))?) else { @@ -423,11 +427,25 @@ impl TrackConsumer { Poll::Ready(Ok(Some(consumer))) } - /// Return the next group in order. + /// Receive the next group available on this track. /// - /// NOTE: This can have gaps if the reader is too slow or there were network slowdowns. + /// Groups may arrive out of order or with gaps due to network conditions. + /// Use `OrderedConsumer` if you need groups in sequence order, + /// skipping those that arrive too late. + pub async fn recv_group(&mut self) -> Result> { + conducer::wait(|waiter| self.poll_recv_group(waiter)).await + } + + /// Deprecated: Use [`recv_group`](Self::recv_group) instead. + #[deprecated(note = "Use recv_group instead")] pub async fn next_group(&mut self) -> Result> { - conducer::wait(|waiter| self.poll_next_group(waiter)).await + self.recv_group().await + } + + /// Deprecated: Use [`poll_recv_group`](Self::poll_recv_group) instead. + #[deprecated(note = "Use poll_recv_group instead")] + pub fn poll_next_group(&mut self, waiter: &conducer::Waiter) -> Poll>> { + self.poll_recv_group(waiter) } /// Poll for the group with the given sequence, without blocking. @@ -485,7 +503,7 @@ use futures::FutureExt; #[cfg(test)] impl TrackConsumer { pub fn assert_group(&mut self) -> GroupConsumer { - self.next_group() + self.recv_group() .now_or_never() .expect("group would have blocked") .expect("would have errored") @@ -494,8 +512,8 @@ impl TrackConsumer { pub fn assert_no_group(&mut self) { assert!( - self.next_group().now_or_never().is_none(), - "next group would not have blocked" + self.recv_group().now_or_never().is_none(), + "recv_group would not have blocked" ); } @@ -753,7 +771,7 @@ mod test { } #[tokio::test] - async fn next_group_finishes_without_waiting_for_gaps() { + async fn recv_group_finishes_without_waiting_for_gaps() { let mut producer = Track::new("test").produce(); producer.create_group(Group { sequence: 1 }).unwrap(); producer.finish_at(1).unwrap(); @@ -762,7 +780,7 @@ mod test { assert_eq!(consumer.assert_group().info.sequence, 1); let done = consumer - .next_group() + .recv_group() .now_or_never() .expect("should not block") .expect("would have errored"); diff --git a/rs/moq-native/tests/backend.rs b/rs/moq-native/tests/backend.rs index 39739f469..c1a4410a1 100644 --- a/rs/moq-native/tests/backend.rs +++ b/rs/moq-native/tests/backend.rs @@ -73,10 +73,10 @@ async fn backend_test(scheme: &str, backend: moq_native::QuicBackend) { .subscribe_track(&Track::new("video")) .expect("subscribe_track failed"); - let mut group_sub = tokio::time::timeout(TIMEOUT, track_sub.next_group()) + let mut group_sub = tokio::time::timeout(TIMEOUT, track_sub.recv_group()) .await - .expect("next_group timed out") - .expect("next_group failed") + .expect("recv_group timed out") + .expect("recv_group failed") .expect("track closed prematurely"); let frame = tokio::time::timeout(TIMEOUT, group_sub.read_frame()) @@ -225,10 +225,10 @@ async fn iroh_connect() { .subscribe_track(&Track::new("video")) .expect("subscribe_track failed"); - let mut group_sub = tokio::time::timeout(TIMEOUT, track_sub.next_group()) + let mut group_sub = tokio::time::timeout(TIMEOUT, track_sub.recv_group()) .await - .expect("next_group timed out") - .expect("next_group failed") + .expect("recv_group timed out") + .expect("recv_group failed") .expect("track closed prematurely"); let frame = tokio::time::timeout(TIMEOUT, group_sub.read_frame()) diff --git a/rs/moq-native/tests/broadcast.rs b/rs/moq-native/tests/broadcast.rs index c1ec6d93e..ecd910492 100644 --- a/rs/moq-native/tests/broadcast.rs +++ b/rs/moq-native/tests/broadcast.rs @@ -92,10 +92,10 @@ async fn broadcast_test(scheme: &str, client_version: Option<&str>, server_versi .expect("subscribe_track failed"); // Read one group. - let mut group_sub = tokio::time::timeout(TIMEOUT, track_sub.next_group()) + let mut group_sub = tokio::time::timeout(TIMEOUT, track_sub.recv_group()) .await - .expect("next_group timed out") - .expect("next_group failed") + .expect("recv_group timed out") + .expect("recv_group failed") .expect("track closed prematurely"); // Read one frame and verify the payload. @@ -432,10 +432,10 @@ async fn broadcast_websocket() { .expect("subscribe_track failed"); // Read one group. - let mut group_sub = tokio::time::timeout(TIMEOUT, track_sub.next_group()) + let mut group_sub = tokio::time::timeout(TIMEOUT, track_sub.recv_group()) .await - .expect("next_group timed out") - .expect("next_group failed") + .expect("recv_group timed out") + .expect("recv_group failed") .expect("track closed prematurely"); // Read one frame and verify the payload. @@ -538,10 +538,10 @@ async fn broadcast_websocket_fallback() { .subscribe_track(&Track::new("video")) .expect("subscribe_track failed"); - let mut group_sub = tokio::time::timeout(TIMEOUT, track_sub.next_group()) + let mut group_sub = tokio::time::timeout(TIMEOUT, track_sub.recv_group()) .await - .expect("next_group timed out") - .expect("next_group failed") + .expect("recv_group timed out") + .expect("recv_group failed") .expect("track closed prematurely"); let frame = tokio::time::timeout(TIMEOUT, group_sub.read_frame()) diff --git a/rs/moq-relay/src/web.rs b/rs/moq-relay/src/web.rs index c2187a2ed..e48fa3721 100644 --- a/rs/moq-relay/src/web.rs +++ b/rs/moq-relay/src/web.rs @@ -295,7 +295,7 @@ async fn serve_fetch( let group = match params.group { FetchGroup::Latest => match track.latest() { Some(sequence) => track.get_group(sequence).await, - None => track.next_group().await, + None => track.recv_group().await, }, FetchGroup::Num(sequence) => track.get_group(sequence).await, }; From fef1bc81db22d225cd8951ba3e8d4878af3239fa Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Mar 2026 13:42:24 -0700 Subject: [PATCH 07/25] Add convert/export modules and subscribe CLI command (#1102) Co-authored-by: Claude Opus 4.6 (1M context) --- CLAUDE.md | 1 + Cargo.lock | 6 +- doc/rs/env/native.md | 7 +- rs/hang/Cargo.toml | 5 +- rs/hang/examples/subscribe.rs | 5 +- rs/hang/src/container/consumer.rs | 1192 +---------------- rs/hang/src/container/mod.rs | 1 + rs/hang/src/container/producer.rs | 10 +- rs/libmoq/src/consume.rs | 12 +- rs/libmoq/src/error.rs | 9 + rs/libmoq/src/publish.rs | 8 +- rs/moq-cli/Cargo.toml | 2 + rs/moq-cli/src/client.rs | 29 - rs/moq-cli/src/main.rs | 227 +++- rs/moq-cli/src/publish.rs | 181 ++- rs/moq-cli/src/server.rs | 51 - rs/moq-cli/src/subscribe.rs | 146 ++ rs/moq-ffi/src/consumer.rs | 8 +- rs/moq-ffi/src/producer.rs | 6 +- rs/moq-mux/Cargo.toml | 5 + rs/moq-mux/src/consumer/container.rs | 103 ++ rs/moq-mux/src/consumer/fmp4.rs | 223 +++ rs/moq-mux/src/consumer/frame.rs | 29 + rs/moq-mux/src/consumer/mod.rs | 11 + rs/moq-mux/src/consumer/muxer.rs | 103 ++ rs/moq-mux/src/consumer/ordered.rs | 1192 +++++++++++++++++ rs/moq-mux/src/convert/fmp4.rs | 511 +++++++ rs/moq-mux/src/convert/hang.rs | 291 ++++ rs/moq-mux/src/convert/mod.rs | 8 + rs/moq-mux/src/convert/test.rs | 418 ++++++ rs/moq-mux/src/lib.rs | 8 +- rs/moq-mux/src/{import => producer}/aac.rs | 0 rs/moq-mux/src/{import => producer}/annexb.rs | 0 rs/moq-mux/src/{import => producer}/av01.rs | 0 rs/moq-mux/src/{import => producer}/avc3.rs | 0 .../src/{import => producer}/decoder.rs | 119 +- rs/moq-mux/src/{import => producer}/fmp4.rs | 242 ++-- rs/moq-mux/src/{import => producer}/hev1.rs | 0 rs/moq-mux/src/{import => producer}/hls.rs | 27 +- rs/moq-mux/src/{import => producer}/mod.rs | 5 +- rs/moq-mux/src/{import => producer}/opus.rs | 0 .../src/{import => producer}/test/av1.mp4 | Bin .../src/{import => producer}/test/bbb.mp4 | Bin .../src/{import => producer}/test/mod.rs | 42 +- .../src/{import => producer}/test/vp9.mp4 | Bin 45 files changed, 3591 insertions(+), 1652 deletions(-) delete mode 100644 rs/moq-cli/src/client.rs delete mode 100644 rs/moq-cli/src/server.rs create mode 100644 rs/moq-cli/src/subscribe.rs create mode 100644 rs/moq-mux/src/consumer/container.rs create mode 100644 rs/moq-mux/src/consumer/fmp4.rs create mode 100644 rs/moq-mux/src/consumer/frame.rs create mode 100644 rs/moq-mux/src/consumer/mod.rs create mode 100644 rs/moq-mux/src/consumer/muxer.rs create mode 100644 rs/moq-mux/src/consumer/ordered.rs create mode 100644 rs/moq-mux/src/convert/fmp4.rs create mode 100644 rs/moq-mux/src/convert/hang.rs create mode 100644 rs/moq-mux/src/convert/mod.rs create mode 100644 rs/moq-mux/src/convert/test.rs rename rs/moq-mux/src/{import => producer}/aac.rs (100%) rename rs/moq-mux/src/{import => producer}/annexb.rs (100%) rename rs/moq-mux/src/{import => producer}/av01.rs (100%) rename rs/moq-mux/src/{import => producer}/avc3.rs (100%) rename rs/moq-mux/src/{import => producer}/decoder.rs (79%) rename rs/moq-mux/src/{import => producer}/fmp4.rs (77%) rename rs/moq-mux/src/{import => producer}/hev1.rs (100%) rename rs/moq-mux/src/{import => producer}/hls.rs (96%) rename rs/moq-mux/src/{import => producer}/mod.rs (80%) rename rs/moq-mux/src/{import => producer}/opus.rs (100%) rename rs/moq-mux/src/{import => producer}/test/av1.mp4 (100%) rename rs/moq-mux/src/{import => producer}/test/bbb.mp4 (100%) rename rs/moq-mux/src/{import => producer}/test/mod.rs (78%) rename rs/moq-mux/src/{import => producer}/test/vp9.mp4 (100%) diff --git a/CLAUDE.md b/CLAUDE.md index ab5d8807a..1fb954138 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -110,6 +110,7 @@ match version { - **`dev`**: Development branch for breaking API changes. PRs with major API changes should target `dev`. - When ready for a new minor/major release, merge `dev` into `main`. - `cargo-semver-checks` enforces this on PRs to `main`. +- When removing a public method on `dev`, mark it `#[deprecated]` first so downstream code gets warnings before the next breaking release. ## Workflow diff --git a/Cargo.lock b/Cargo.lock index 2dea3f174..94835ee13 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2109,11 +2109,11 @@ dependencies = [ "anyhow", "buf-list", "bytes", - "conducer", "derive_more", "hex", "lazy_static", "moq-lite", + "moq-mux", "moq-native", "regex", "serde", @@ -3249,10 +3249,12 @@ dependencies = [ "anyhow", "axum", "axum-server", + "base64 0.22.1", "clap", "hang", "moq-mux", "moq-native", + "mp4-atom", "rustls", "sd-notify", "tokio", @@ -3326,6 +3328,7 @@ dependencies = [ "base64 0.22.1", "buf-list", "bytes", + "conducer", "derive_more", "h264-parser", "hang", @@ -3337,6 +3340,7 @@ dependencies = [ "reqwest 0.12.28", "scuffle-av1", "scuffle-h265", + "thiserror 2.0.18", "tokio", "tracing", "url", diff --git a/doc/rs/env/native.md b/doc/rs/env/native.md index 85f4728fe..bc9393b8e 100644 --- a/doc/rs/env/native.md +++ b/doc/rs/env/native.md @@ -123,17 +123,18 @@ See the full [subscribe.rs](https://github.com/moq-dev/moq/blob/main/rs/hang/exa ## Reading Frames -Subscribe to a media track and read frames using [`OrderedConsumer`](https://docs.rs/hang/latest/hang/container/struct.OrderedConsumer.html): +Subscribe to a media track and read frames using [`OrderedConsumer`](https://docs.rs/moq-mux/latest/moq_mux/consumer/struct.OrderedConsumer.html): ```rust let track_consumer = broadcast.subscribe_track(&track); -let mut ordered = hang::container::OrderedConsumer::new( +let mut ordered = moq_mux::consumer::OrderedConsumer::new( track_consumer, + moq_mux::consumer::Legacy, Duration::from_millis(500), // max latency before skipping groups ); while let Some(frame) = ordered.read().await? { - // frame.timestamp, frame.keyframe, frame.payload + // frame.timestamp, frame.is_keyframe(), frame.payload } ``` diff --git a/rs/hang/Cargo.toml b/rs/hang/Cargo.toml index 55c75fc34..2cfee599e 100644 --- a/rs/hang/Cargo.toml +++ b/rs/hang/Cargo.toml @@ -16,7 +16,6 @@ categories = ["multimedia", "network-programming", "web-programming"] [dependencies] buf-list = "1" bytes = "1" -conducer = { workspace = true } hex = "0.4" lazy_static = "1" moq-lite = { workspace = true, features = ["serde"] } @@ -35,6 +34,8 @@ features = ["from", "display", "debug"] [dev-dependencies] anyhow = "1" -moq-native = { path = "../moq-native" } +moq-mux = { workspace = true } +moq-native = { workspace = true, default-features = true } +tokio = { workspace = true, features = ["test-util"] } tracing = "0.1" url = "2" diff --git a/rs/hang/examples/subscribe.rs b/rs/hang/examples/subscribe.rs index f48c079a3..1e936cbee 100644 --- a/rs/hang/examples/subscribe.rs +++ b/rs/hang/examples/subscribe.rs @@ -77,7 +77,10 @@ async fn run_subscribe(mut consumer: moq_lite::OriginConsumer) -> anyhow::Result }; let track_consumer = broadcast.subscribe_track(&track)?; - let mut ordered = hang::container::OrderedConsumer::new(track_consumer, Duration::from_millis(500)); + + // Skip over groups where all frames are older than 500ms to maintain low latency. + let mut ordered = + moq_mux::consumer::OrderedConsumer::new(track_consumer, moq_mux::consumer::Legacy, Duration::from_millis(500)); // Read frames in presentation order. while let Some(frame) = ordered.read().await? { diff --git a/rs/hang/src/container/consumer.rs b/rs/hang/src/container/consumer.rs index e7e18c1d4..924fc03cb 100644 --- a/rs/hang/src/container/consumer.rs +++ b/rs/hang/src/container/consumer.rs @@ -1,38 +1,24 @@ -use std::collections::VecDeque; -use std::task::{Poll, ready}; - -use buf_list::BufList; - use super::{Frame, Timestamp}; -use crate::Error; +use buf_list::BufList; /// A frame returned by [`OrderedConsumer::read()`] with group context. +#[deprecated(note = "use moq_mux::consumer::OrderedFrame instead")] #[derive(Clone, Debug)] pub struct OrderedFrame { - /// The presentation timestamp for this frame. pub timestamp: Timestamp, - - /// The encoded media data for this frame, split into chunks. pub payload: BufList, - - /// The group sequence number this frame belongs to. pub group: u64, - - /// The frame index within the group (0 = first frame in the group). - /// - /// With duration-based grouping (e.g. audio), the first frame is not - /// necessarily a keyframe — it only denotes position within the group. pub index: usize, } +#[allow(deprecated)] impl OrderedFrame { - /// Returns true if this is the first frame in the group (index 0). pub fn is_keyframe(&self) -> bool { self.index == 0 } } -/// Lossy conversion: discards ordering metadata (`group` and `frame` fields). +#[allow(deprecated)] impl From for Frame { fn from(ordered: OrderedFrame) -> Self { Frame { @@ -42,1172 +28,22 @@ impl From for Frame { } } -/// A consumer for hang-formatted media tracks with timestamp reordering. -/// -/// This wraps a `moq_lite::TrackConsumer` and adds hang-specific functionality -/// like timestamp decoding, latency management, and frame buffering. +/// Deprecated: use `moq_mux::consumer::OrderedConsumer` instead. /// -/// ## Latency Management -/// -/// The consumer can skip groups that are too far behind to maintain low latency. -/// Configure the maximum acceptable delay through the consumer's latency settings. +/// This stub exists only to provide a deprecation warning. +/// The implementation has been moved to `moq_mux::consumer::OrderedConsumer`. +#[deprecated(note = "use moq_mux::consumer::OrderedConsumer instead")] pub struct OrderedConsumer { - pub track: moq_lite::TrackConsumer, - - // The current group that we want to read from - current: u64, - - // Groups that we are monitoring, sorted by sequence ascending. - pending: VecDeque, - - // When true, we haven't returned a frame yet and need to select the first group. - // We wait until we have at least one frame before finalizing `current` - startup: bool, - - // The maximum buffer size before skipping a group. - max_latency: std::time::Duration, + _private: (), } +#[allow(deprecated)] impl OrderedConsumer { - /// Create a new OrderedConsumer wrapping the given moq-lite consumer. - pub fn new(track: moq_lite::TrackConsumer, max_latency: std::time::Duration) -> Self { - Self { - track, - current: 0, - pending: VecDeque::new(), - startup: true, - max_latency, - } - } - - /// Read the next frame from the track. - /// - /// This method handles timestamp decoding, group ordering, and latency management - /// automatically. It will skip groups that are too far behind to maintain the - /// configured latency target. - /// - /// Returns `None` when the track has ended. - pub async fn read(&mut self) -> Result, Error> { - conducer::wait(|waiter| self.poll_read(waiter)).await - } - - /// Poll-based implementation of the read loop. - /// - /// Uses a single waiter that gets registered on all relevant conducer channels, - /// avoiding the need for `tokio::select!` or `FuturesUnordered`. - fn poll_read(&mut self, waiter: &conducer::Waiter) -> Poll, Error>> { - // Grab any new groups from the track, recording whether the track is finished. - let finished = self.poll_read_finish(waiter)?.is_ready(); - - // On startup, we want to poll every pending group and advance self.current to the first with a frame. - if self.startup { - // NOTE: We loop in ascending order, so earlier groups will win the race. - for (i, group) in self.pending.iter_mut().enumerate() { - // We call poll_min_timestamp to try to buffer at least one frame per group. - // This returns Ready(Ok) if there is a buffered frame. - if !matches!(group.poll_min_timestamp(waiter), Poll::Ready(Ok(_))) { - continue; - } - - // Start reading from this group and skip any previous groups. - self.current = group.info.sequence; - self.startup = false; - self.pending.drain(0..i); - break; - } - } - - loop { - // Return the next frame from the current group if possible. - // If the current group is finished or errored, advance to the next group. - while let Some(group) = self.pending.front_mut() - && group.info.sequence <= self.current - { - match group.poll_read(waiter) { - Poll::Ready(Ok(Some(frame))) => return Poll::Ready(Ok(Some(frame))), - // Still blocked on this group, don't skip it yet. - Poll::Pending => break, - Poll::Ready(Err(e)) => { - tracing::warn!(error = ?e, "error reading current group, skipping"); - } - // No more frames, advance to next group. - Poll::Ready(Ok(None)) => {} - } - - self.pending.pop_front(); - self.current += 1 - } - - // Loop in ascending order to get the min, avoiding spurious wakeups. - let mut min_timestamp = std::time::Duration::MAX; - let mut min_idx = None; - - for (i, group) in self.pending.iter_mut().enumerate() { - if group.info.sequence <= self.current { - continue; - } - - if let Poll::Ready(Ok(ts)) = group.poll_min_timestamp(waiter) { - min_timestamp = min_timestamp.min(ts.into()); - min_idx = Some(i); - break; // We know future groups won't be older than this. - } - } - - // Loop in descending order to get the max, avoiding spurious wakeups. - let mut max_timestamp = std::time::Duration::ZERO; - for group in self.pending.iter_mut().rev() { - if group.info.sequence <= self.current { - break; - } - - if let Poll::Ready(Ok(ts)) = group.poll_max_timestamp(waiter) { - max_timestamp = max_timestamp.max(ts.into()); - break; // We know older groups won't be newer than this. - } - } - - if let Some(new_idx) = min_idx - && max_timestamp.saturating_sub(min_timestamp) >= self.max_latency - { - self.pending.drain(0..new_idx); - let new_current = self.pending.front().map(|g| g.info.sequence).unwrap(); - - tracing::debug!(old = self.current, new = new_current, "skipping slow groups"); - - self.current = new_current; - continue; - } - - if finished && self.pending.is_empty() { - return Poll::Ready(Ok(None)); - } - - return Poll::Pending; - } - } - - // Reads any new groups from the track until we're completely finished. - // - // Returns Pending until all groups have been consumed. - fn poll_read_finish(&mut self, waiter: &conducer::Waiter) -> Poll> { - loop { - let Some(group) = ready!(self.track.poll_recv_group(waiter)?) else { - // Track is finished. - return Poll::Ready(Ok(())); - }; - - let reader = GroupBuffer::new(group); - if reader.group.info.sequence < self.current { - tracing::debug!( - old = ?reader.group.info.sequence, - current = ?self.current, - "skipping old group" - ); - continue; - } - - let idx = self - .pending - .partition_point(|g| g.group.info.sequence < reader.group.info.sequence); - self.pending.insert(idx, reader); - } - } - - /// Set the maximum latency tolerance for this consumer. - /// - /// Groups with timestamps older than `max_timestamp - max_latency` will be skipped. - pub fn set_max_latency(&mut self, max: std::time::Duration) { - self.max_latency = max; - } - - /// Wait until the track is closed. - pub async fn closed(&self) -> Result<(), Error> { - Ok(self.track.closed().await?) - } -} - -impl From for moq_lite::TrackConsumer { - fn from(inner: OrderedConsumer) -> Self { - inner.track - } -} - -impl std::ops::Deref for OrderedConsumer { - type Target = moq_lite::TrackConsumer; - - fn deref(&self) -> &Self::Target { - &self.track - } -} - -/// Internal reader for a group of frames. -/// -/// Handles two-phase frame reading (get FrameConsumer, then read all data), -/// timestamp parsing, and min/max timestamp tracking for latency decisions. -struct GroupBuffer { - group: moq_lite::GroupConsumer, - - // The current frame index within the group. - index: usize, - - // Read frames that haven't been consumed yet. - buffered: VecDeque, - - // The minimum timestamp in the group. - min_timestamp: Option, - - // The maximum timestamp in the group. - max_timestamp: Option, -} - -impl GroupBuffer { - fn new(group: moq_lite::GroupConsumer) -> Self { - Self { - group, - index: 0, - buffered: VecDeque::new(), - max_timestamp: None, - min_timestamp: None, - } - } - - /// Poll for the next frame from this group. - pub fn poll_read(&mut self, waiter: &conducer::Waiter) -> Poll, Error>> { - if let Some(frame) = self.buffered.pop_front() { - return Poll::Ready(Ok(Some(frame))); - } - - match ready!(self.buffer_one(waiter)?) { - true => Poll::Ready(Ok(Some(self.buffered.pop_front().unwrap()))), - false => Poll::Ready(Ok(None)), - } - } - - // Add one more frame to the buffer if possible. - // - // Returns false if the track is finished. - fn buffer_once(&mut self, waiter: &conducer::Waiter) -> Poll> { - let Some(chunks) = ready!(self.group.poll_read_frame_chunks(waiter)?) else { - return Poll::Ready(Ok(false)); - }; - - let mut payload = BufList::from_iter(chunks); - let timestamp = Timestamp::decode(&mut payload)?; - - self.min_timestamp = Some(match self.min_timestamp { - Some(existing) => existing.min(timestamp), - None => timestamp, - }); - - self.max_timestamp = Some(match self.max_timestamp { - Some(existing) => existing.max(timestamp), - None => timestamp, - }); - - let index = self.index; - self.index += 1; - - self.buffered.push_back(OrderedFrame { - timestamp, - payload, - group: self.group.info.sequence, - index, - }); - - Poll::Ready(Ok(true)) - } - - fn buffer_one(&mut self, waiter: &conducer::Waiter) -> Poll> { - if self.buffered.is_empty() { - self.buffer_once(waiter) - } else { - Poll::Ready(Ok(true)) - } - } - - fn buffer_all(&mut self, waiter: &conducer::Waiter) -> Poll> { - while ready!(self.buffer_once(waiter)?) {} - Poll::Ready(Ok(())) - } - - /// Poll for the maximum timestamp in this group. - pub fn poll_max_timestamp(&mut self, waiter: &conducer::Waiter) -> Poll> { - // Keep reading more frames just to advance the max timestamp. - let _ = self.buffer_all(waiter)?; - - if let Some(max) = self.max_timestamp { - return Poll::Ready(Ok(max)); - } - - if let Poll::Ready(_frames) = self.group.poll_finished(waiter)? { - return Poll::Ready(Err(Error::EmptyGroup)); - } - - Poll::Pending - } - - pub fn poll_min_timestamp(&mut self, waiter: &conducer::Waiter) -> Poll> { - let _ = self.buffer_one(waiter)?; - - if let Some(min) = self.min_timestamp { - return Poll::Ready(Ok(min)); - } - - if let Poll::Ready(_frames) = self.group.poll_finished(waiter)? { - return Poll::Ready(Err(Error::EmptyGroup)); - } - - Poll::Pending - } -} - -impl std::ops::Deref for GroupBuffer { - type Target = moq_lite::GroupConsumer; - - fn deref(&self) -> &Self::Target { - &self.group - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::time::Duration; - - use bytes::Bytes; - - fn ts(micros: u64) -> Timestamp { - Timestamp::from_micros(micros).unwrap() - } - - /// Write a finished group with explicit sequence and timestamps. - fn write_group(track: &mut moq_lite::TrackProducer, sequence: u64, timestamps: &[Timestamp]) { - let mut group = track.create_group(moq_lite::Group { sequence }).unwrap(); - for ×tamp in timestamps { - let frame = Frame { - timestamp, - payload: BufList::from_iter(vec![Bytes::from_static(&[0xDE, 0xAD])]), - }; - frame.encode(&mut group).unwrap(); - } - group.finish().unwrap(); - } - - /// Drain all available frames with a per-read timeout. - async fn read_all(consumer: &mut OrderedConsumer) -> Result, crate::Error> { - let mut frames = Vec::new(); - loop { - match tokio::time::timeout(Duration::from_millis(200), consumer.read()).await { - Ok(Ok(Some(frame))) => frames.push(frame), - Ok(Ok(None)) => break, - Ok(Err(e)) => return Err(e), - Err(_) => panic!( - "read_all: OrderedConsumer::read timed out after 200ms ({} frames collected so far)", - frames.len() - ), - } - } - Ok(frames) - } - - // ---- Basic Reading ---- - - #[tokio::test] - async fn read_single_group() { - let mut track = moq_lite::Track::new("test").produce(); - let consumer_track = track.consume(); - let mut consumer = OrderedConsumer::new(consumer_track, Duration::from_millis(500)); - - write_group(&mut track, 0, &[ts(0)]); - track.finish().unwrap(); - - let frames = read_all(&mut consumer).await.unwrap(); - assert_eq!(frames.len(), 1); - assert_eq!(frames[0].timestamp, ts(0)); - assert_eq!(frames[0].index, 0); - - // Next read returns None (track ended) - assert!(consumer.read().await.unwrap().is_none()); - } - - #[tokio::test] - async fn read_multiple_frames_single_group() { - let mut track = moq_lite::Track::new("test").produce(); - let consumer_track = track.consume(); - let mut consumer = OrderedConsumer::new(consumer_track, Duration::from_millis(500)); - - write_group(&mut track, 0, &[ts(0), ts(33_000), ts(66_000)]); - track.finish().unwrap(); - - let frames = read_all(&mut consumer).await.unwrap(); - assert_eq!(frames.len(), 3); - assert_eq!(frames[0].timestamp, ts(0)); - assert_eq!(frames[1].timestamp, ts(33_000)); - assert_eq!(frames[2].timestamp, ts(66_000)); - - assert_eq!(frames[0].index, 0); - assert_eq!(frames[1].index, 1); - assert_eq!(frames[2].index, 2); - } - - #[tokio::test] - async fn read_multiple_groups_within_latency() { - let mut track = moq_lite::Track::new("test").produce(); - let consumer_track = track.consume(); - let mut consumer = OrderedConsumer::new(consumer_track, Duration::from_millis(500)); - - // 5 groups, 20ms spacing. Total span = 80ms, well within 500ms latency. - for i in 0..5u64 { - write_group(&mut track, i, &[ts(i * 20_000)]); - } - track.finish().unwrap(); - - let frames = read_all(&mut consumer).await.unwrap(); - assert_eq!(frames.len(), 5); - } - - // ---- Latency Skipping ---- - // - // These tests verify that the poll-based latency skip logic correctly - // promotes pending groups to current when the timestamp span exceeds - // max_latency. - - #[tokio::test] - async fn latency_skip_delivers_recent_groups() { - tokio::time::pause(); - let mut track = moq_lite::Track::new("test").produce(); - let consumer_track = track.consume(); - let mut consumer = OrderedConsumer::new(consumer_track, Duration::from_millis(100)); - - // Group 0: 5 frames, NOT finished (blocks consumer) - let mut group0 = track.create_group(moq_lite::Group { sequence: 0 }).unwrap(); - for f in 0..5u64 { - Frame { - timestamp: ts(f * 2_000), - payload: BufList::from_iter(vec![Bytes::from_static(&[0xDE, 0xAD])]), - } - .encode(&mut group0) - .unwrap(); - } - - // Groups 1-19: finished, 15ms spacing, 5 frames each - for g in 1..20u64 { - let timestamps: Vec<_> = (0..5).map(|f| ts(g * 15_000 + f * 2_000)).collect(); - write_group(&mut track, g, ×tamps); - } - track.finish().unwrap(); - - // Finish group 0 after consumer has had time to accumulate pending groups - let finisher = tokio::spawn(async move { - tokio::time::sleep(Duration::from_millis(50)).await; - group0.finish().unwrap(); - }); - - let frames = read_all(&mut consumer).await.unwrap(); - // Group 0's 5 frames + some later groups (earlier ones skipped by latency) - assert!(frames.len() >= 25, "Expected >= 25 frames, got {}", frames.len()); - finisher.await.expect("finisher task panicked"); - } - - #[tokio::test] - async fn zero_latency_skips_aggressively() { - tokio::time::pause(); - let mut track = moq_lite::Track::new("test").produce(); - let consumer_track = track.consume(); - let mut consumer = OrderedConsumer::new(consumer_track, Duration::ZERO); - - // Group 0: 1 frame at a HIGH timestamp, NOT finished. - // This makes the cutoff high (max_timestamp + 0 = 400ms), so - // buffer_until blocks for groups whose timestamps are < 400ms. - let mut group0 = track.create_group(moq_lite::Group { sequence: 0 }).unwrap(); - Frame { - timestamp: ts(400_000), - payload: BufList::from_iter(vec![Bytes::from_static(&[0xDE, 0xAD])]), - } - .encode(&mut group0) - .unwrap(); - - // Groups 1-9: finished, 50ms spacing, 3 frames each - // Groups 1-7 have max timestamps < 400ms (blocked by buffer_until) - // Group 8+ have timestamps >= 400ms (trigger latency skip) - for g in 1..10u64 { - let timestamps: Vec<_> = (0..3).map(|f| ts(g * 50_000 + f * 5_000)).collect(); - write_group(&mut track, g, ×tamps); - } - track.finish().unwrap(); - - let finisher = tokio::spawn(async move { - tokio::time::sleep(Duration::from_millis(50)).await; - group0.finish().unwrap(); - }); - - let frames = read_all(&mut consumer).await.unwrap(); - // The latency skip bridges past the blocking group 0 to the nearest - // pending group with data. All subsequent finished groups are delivered - // instantly. Group 0's 1 frame + groups 1-9 (3 frames each) = 28. - assert_eq!(frames.len(), 28, "Expected group 0 frame + groups 1-9"); - assert!(!frames.is_empty(), "Expected at least some frames"); - finisher.await.expect("finisher task panicked"); - } - - #[tokio::test] - async fn latency_skip_correctness() { - tokio::time::pause(); - let mut track = moq_lite::Track::new("test").produce(); - let consumer_track = track.consume(); - let mut consumer = OrderedConsumer::new(consumer_track, Duration::from_millis(100)); - - // Group 0: 1 frame, NOT finished - let mut group0 = track.create_group(moq_lite::Group { sequence: 0 }).unwrap(); - Frame { - timestamp: ts(0), - payload: BufList::from_iter(vec![Bytes::from_static(&[0xDE, 0xAD])]), - } - .encode(&mut group0) - .unwrap(); - - // Groups 1-9: 30ms spacing, 1 frame each - for g in 1..10u64 { - write_group(&mut track, g, &[ts(g * 30_000)]); - } - track.finish().unwrap(); - - let finisher = tokio::spawn(async move { - tokio::time::sleep(Duration::from_millis(50)).await; - group0.finish().unwrap(); - }); - - let frames = read_all(&mut consumer).await.unwrap(); - assert!(!frames.is_empty(), "Expected at least some frames"); - - // The latency skip bridges past the blocking group 0 to the nearest - // pending group with data. All subsequent groups are delivered since - // they can be read instantly (already finished). Group 0's frame (ts=0) - // is returned before the skip, then groups 1-9 after. - assert_eq!(frames.len(), 10, "Expected group 0 frame + groups 1-9"); - assert_eq!(frames[0].timestamp, ts(0)); - - // Groups should be delivered in sequence order - for i in 1..10u64 { - assert_eq!(frames[i as usize].timestamp, ts(i * 30_000)); - } - finisher.await.expect("finisher task panicked"); - } - - // ---- Group Ordering ---- - - #[tokio::test] - async fn groups_delivered_in_sequence_order() { - tokio::time::pause(); - let mut track = moq_lite::Track::new("test").produce(); - let consumer_track = track.consume(); - let mut consumer = OrderedConsumer::new(consumer_track, Duration::from_millis(500)); - - // Group 0: 1 frame, NOT finished (blocks consumer, lets groups 2 and 1 accumulate in pending) - let mut group0 = track.create_group(moq_lite::Group { sequence: 0 }).unwrap(); - Frame { - timestamp: ts(0), - payload: BufList::from_iter(vec![Bytes::from_static(&[0xDE, 0xAD])]), - } - .encode(&mut group0) - .unwrap(); - - // Write groups 2 then 1 (out of sequence order) - write_group(&mut track, 2, &[ts(60_000)]); - write_group(&mut track, 1, &[ts(30_000)]); - track.finish().unwrap(); - - // Finish group 0 so the consumer can proceed to pending groups - let finisher = tokio::spawn(async move { - tokio::time::sleep(Duration::from_millis(10)).await; - group0.finish().unwrap(); - }); - - let frames = read_all(&mut consumer).await.unwrap(); - assert_eq!(frames.len(), 3); - - // Pending queue sorts by sequence, so delivery order is 0, 1, 2 - assert_eq!(frames[0].timestamp, ts(0)); - assert_eq!(frames[1].timestamp, ts(30_000)); - assert_eq!(frames[2].timestamp, ts(60_000)); - finisher.await.expect("finisher task panicked"); - } - - #[tokio::test] - async fn adjacent_group_flushed_immediately() { - let mut track = moq_lite::Track::new("test").produce(); - let consumer_track = track.consume(); - let mut consumer = OrderedConsumer::new(consumer_track, Duration::from_millis(500)); - - write_group(&mut track, 0, &[ts(0)]); - write_group(&mut track, 1, &[ts(30_000)]); - track.finish().unwrap(); - - let frames = read_all(&mut consumer).await.unwrap(); - assert_eq!(frames.len(), 2); - assert_eq!(frames[0].timestamp, ts(0)); - assert_eq!(frames[1].timestamp, ts(30_000)); - } - - // ---- B-frames ---- - - #[tokio::test] - async fn bframes_within_group() { - let mut track = moq_lite::Track::new("test").produce(); - let consumer_track = track.consume(); - let mut consumer = OrderedConsumer::new(consumer_track, Duration::from_millis(500)); - - // B-frame decode order: timestamps [0, 66ms, 33ms] - write_group(&mut track, 0, &[ts(0), ts(66_000), ts(33_000)]); - track.finish().unwrap(); - - let frames = read_all(&mut consumer).await.unwrap(); - assert_eq!(frames.len(), 3); - // Delivered in write order (decode order), not presentation order - assert_eq!(frames[0].timestamp, ts(0)); - assert_eq!(frames[1].timestamp, ts(66_000)); - assert_eq!(frames[2].timestamp, ts(33_000)); - } - - // ---- Track Lifecycle ---- - - #[tokio::test] - async fn empty_track_returns_none() { - tokio::time::pause(); - let mut track = moq_lite::Track::new("test").produce(); - let consumer_track = track.consume(); - let mut consumer = OrderedConsumer::new(consumer_track, Duration::from_millis(500)); - - track.finish().unwrap(); - - let result = tokio::time::timeout(Duration::from_millis(200), consumer.read()).await; - match result { - Ok(Ok(None)) => {} // expected: track ended - Ok(Ok(Some(_))) => panic!("expected None for empty track, got Some"), - Ok(Err(e)) => panic!("expected None for empty track, got error: {e}"), - Err(_) => panic!("should not hang on empty track"), - } - } - - #[tokio::test] - async fn track_closed_with_error() { - tokio::time::pause(); - let mut track = moq_lite::Track::new("test").produce(); - let consumer_track = track.consume(); - let mut consumer = OrderedConsumer::new(consumer_track, Duration::from_millis(500)); - - write_group(&mut track, 0, &[ts(0)]); - track.abort(moq_lite::Error::Cancel).unwrap(); - - // Consumer should not hang; it should return frames or error gracefully - let result = tokio::time::timeout(Duration::from_millis(500), async { - let mut frames = Vec::new(); - while let Ok(Some(frame)) = consumer.read().await { - frames.push(frame); - } - frames - }) - .await; - - assert!(result.is_ok(), "Consumer should not hang after track error"); - } - - #[tokio::test] - async fn closed_resolves_when_track_ends() { - tokio::time::pause(); - let mut track = moq_lite::Track::new("test").produce(); - let consumer_track = track.consume(); - let consumer = OrderedConsumer::new(consumer_track, Duration::from_millis(500)); - - // closed() should not resolve yet - assert!( - tokio::time::timeout(Duration::from_millis(50), consumer.closed()) - .await - .is_err() - ); - - // finish() + drop triggers the Closed/Dropped state that closed() waits for - track.finish().unwrap(); - drop(track); - - // closed() should resolve now - tokio::time::timeout(Duration::from_millis(200), consumer.closed()) - .await - .expect("timeout expired waiting for closed()") - .expect("consumer.closed() returned an error"); - } - - // ---- Gap Recovery ---- - - #[tokio::test] - async fn gap_in_group_sequence_recovery() { - let mut track = moq_lite::Track::new("test").produce(); - let consumer_track = track.consume(); - let mut consumer = OrderedConsumer::new(consumer_track, Duration::from_millis(100)); - - // Groups 0, 1 then skip 2, write 3-6 - write_group(&mut track, 0, &[ts(0), ts(20_000)]); - write_group(&mut track, 1, &[ts(40_000), ts(60_000)]); - // Gap at group 2 - write_group(&mut track, 3, &[ts(120_000), ts(140_000)]); - write_group(&mut track, 4, &[ts(160_000), ts(180_000)]); - write_group(&mut track, 5, &[ts(200_000), ts(220_000)]); - write_group(&mut track, 6, &[ts(240_000), ts(260_000)]); - track.finish().unwrap(); - - // Consumer must not deadlock on the missing group 2 - let frames = read_all(&mut consumer).await.unwrap(); - assert!(frames.len() >= 4, "Expected >= 4 frames, got {}", frames.len()); - } - - #[tokio::test] - async fn gap_at_start_of_sequence() { - let mut track = moq_lite::Track::new("test").produce(); - let consumer_track = track.consume(); - let mut consumer = OrderedConsumer::new(consumer_track, Duration::from_millis(80)); - - // First group at sequence 5 (simulating joining mid-stream), gap at 6 - write_group(&mut track, 5, &[ts(0), ts(20_000)]); - write_group(&mut track, 7, &[ts(80_000), ts(100_000)]); - write_group(&mut track, 8, &[ts(120_000), ts(140_000)]); - write_group(&mut track, 9, &[ts(160_000), ts(180_000)]); - track.finish().unwrap(); - - let frames = read_all(&mut consumer).await.unwrap(); - assert!(frames.len() >= 4, "Expected >= 4 frames, got {}", frames.len()); - } - - // ---- Frame Decoding ---- - - #[tokio::test] - async fn frame_timestamp_and_index_decoding() { - let mut track = moq_lite::Track::new("test").produce(); - let consumer_track = track.consume(); - let mut consumer = OrderedConsumer::new(consumer_track, Duration::from_millis(500)); - - write_group(&mut track, 0, &[ts(0), ts(33_333), ts(66_666)]); - track.finish().unwrap(); - - let frames = read_all(&mut consumer).await.unwrap(); - assert_eq!(frames.len(), 3); - - assert_eq!(frames[0].timestamp, ts(0)); - assert_eq!(frames[0].index, 0); - - assert_eq!(frames[1].timestamp, ts(33_333)); - assert_eq!(frames[1].index, 1); - - assert_eq!(frames[2].timestamp, ts(66_666)); - assert_eq!(frames[2].index, 2); - } - - #[tokio::test] - async fn frame_payload_preserved() { - let mut track = moq_lite::Track::new("test").produce(); - let consumer_track = track.consume(); - let mut consumer = OrderedConsumer::new(consumer_track, Duration::from_millis(500)); - - let payload_bytes = vec![0x01, 0x02, 0x03, 0x04, 0x05]; - let mut group = track.create_group(moq_lite::Group { sequence: 0 }).unwrap(); - Frame { - timestamp: ts(0), - payload: BufList::from_iter(vec![Bytes::from(payload_bytes.clone())]), - } - .encode(&mut group) - .unwrap(); - group.finish().unwrap(); - track.finish().unwrap(); - - let frames = read_all(&mut consumer).await.unwrap(); - assert_eq!(frames.len(), 1); - - use bytes::Buf; - let mut received = Vec::new(); - let mut payload = frames[0].payload.clone(); - while payload.has_remaining() { - received.push(payload.get_u8()); - } - assert_eq!(received, payload_bytes); - } - - // ---- Regression ---- - - /// Regression test for de92d2c7: the old select!-based implementation had an - /// infinite loop when a pending group had buffered frames from a prior - /// (dropped) buffer_until call. - /// - /// The poll-based rewrite avoids this by design: frames are only read on-demand, - /// and buffered frames are consumed before polling for new ones. - #[tokio::test] - async fn no_infinite_loop_with_buffered_frames() { - tokio::time::pause(); - let mut track = moq_lite::Track::new("test").produce(); - let consumer_track = track.consume(); - let mut consumer = OrderedConsumer::new(consumer_track, Duration::from_secs(10)); - - // Group 0: 1 frame, NOT finished - let mut group0 = track.create_group(moq_lite::Group { sequence: 0 }).unwrap(); - Frame { - timestamp: ts(0), - payload: BufList::from_iter(vec![Bytes::from_static(&[0xDE, 0xAD])]), - } - .encode(&mut group0) - .unwrap(); - - // Group 1: finished (buffer_until will buffer its frames) - write_group(&mut track, 1, &[ts(100_000)]); - - let finisher = tokio::spawn(async move { - // After consumer has buffered group 1's frames via buffer_until... - tokio::time::sleep(Duration::from_millis(20)).await; - // Write group 2: recv_group fires, drops current buffer_until for group 1 - write_group(&mut track, 2, &[ts(200_000)]); - // Then finish group 0: consumer proceeds, re-creates buffer_until for group 1 - tokio::time::sleep(Duration::from_millis(20)).await; - group0.finish().unwrap(); - track.finish().unwrap(); - }); - - // Must complete within 2 seconds (with the bug, this would hang) - let frames = tokio::time::timeout(Duration::from_secs(2), async { - let mut frames = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame); - } - frames - }) - .await - .expect("consumer hung — possible infinite loop regression"); - - assert_eq!(frames.len(), 3); - finisher.await.expect("finisher task panicked"); - } - - // ---- Edge Cases ---- - - #[tokio::test] - async fn large_timestamps() { - let mut track = moq_lite::Track::new("test").produce(); - let consumer_track = track.consume(); - let mut consumer = OrderedConsumer::new(consumer_track, Duration::from_secs(3700)); - - // 1 hour = 3,600,000,000 microseconds - let one_hour = 3_600_000_000u64; - write_group(&mut track, 0, &[ts(one_hour)]); - track.finish().unwrap(); - - let frames = read_all(&mut consumer).await.unwrap(); - assert_eq!(frames.len(), 1); - assert_eq!(frames[0].timestamp, ts(one_hour)); - assert_eq!(frames[0].timestamp.as_micros(), one_hour as u128); + pub fn new(_track: moq_lite::TrackConsumer, _max_latency: std::time::Duration) -> Self { + panic!("hang::container::OrderedConsumer has been moved to moq_mux::consumer::OrderedConsumer") } - #[tokio::test] - async fn set_max_latency_changes_behavior() { - let mut track = moq_lite::Track::new("test").produce(); - let consumer_track = track.consume(); - let mut consumer = OrderedConsumer::new(consumer_track, Duration::from_secs(10)); - - write_group(&mut track, 0, &[ts(0)]); - track.finish().unwrap(); - - // Read with initial large latency - let frame = consumer.read().await.unwrap().unwrap(); - assert_eq!(frame.timestamp, ts(0)); - - // Change latency — verify it doesn't panic and consumer still works - consumer.set_max_latency(Duration::from_millis(100)); - - // Track is already finished, so next read returns None - assert!(consumer.read().await.unwrap().is_none()); - } - - /// Verify max_timestamp tracks the true maximum through B-frame reordering. - /// - /// With B-frame decode order [0, 66ms, 33ms], the bug assigned max_timestamp = 33ms - /// (last frame) instead of 66ms (true max). This lowered the latency cutoff, causing - /// premature skipping of subsequent groups under tight latency settings. - /// - /// Setup: group 0 is unfinished with B-frames, group 1 at ts(100ms), latency = 40ms. - /// Bug: cutoff = 33ms + 40ms = 73ms → group 1's buffer_until sees 100ms >= 73ms → skip - /// Fix: cutoff = 66ms + 40ms = 106ms → 100ms < 106ms → no skip, all groups delivered - #[tokio::test] - async fn max_timestamp_tracks_through_bframes() { - tokio::time::pause(); - let mut track = moq_lite::Track::new("test").produce(); - let consumer_track = track.consume(); - let mut consumer = OrderedConsumer::new(consumer_track, Duration::from_millis(40)); - - // Group 0: B-frame decode order [0, 66ms, 33ms], NOT finished (blocks consumer) - let mut group0 = track.create_group(moq_lite::Group { sequence: 0 }).unwrap(); - for ×tamp in &[ts(0), ts(66_000), ts(33_000)] { - Frame { - timestamp, - payload: BufList::from_iter(vec![Bytes::from_static(&[0xDE, 0xAD])]), - } - .encode(&mut group0) - .unwrap(); - } - - // Group 1: finished, at ts(100ms) - write_group(&mut track, 1, &[ts(100_000)]); - track.finish().unwrap(); - - // Finish group 0 after consumer has had time to accumulate pending groups - let finisher = tokio::spawn(async move { - tokio::time::sleep(Duration::from_millis(50)).await; - group0.finish().unwrap(); - }); - - let frames = tokio::time::timeout(Duration::from_secs(2), async { - let mut frames = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame); - } - frames - }) - .await - .expect("consumer hung — max_timestamp regression"); - - assert_eq!(frames.len(), 4, "Expected all 4 frames, got {}", frames.len()); - assert_eq!(frames[0].timestamp, ts(0)); - assert_eq!(frames[1].timestamp, ts(66_000)); - assert_eq!(frames[2].timestamp, ts(33_000)); - assert_eq!(frames[3].timestamp, ts(100_000)); - finisher.await.expect("finisher task panicked"); - } - - // ---- Startup Behavior ---- - - #[tokio::test] - async fn startup_selects_earliest_group() { - tokio::time::pause(); - let mut track = moq_lite::Track::new("test").produce(); - let consumer_track = track.consume(); - // max_latency = 100ms. - let mut consumer = OrderedConsumer::new(consumer_track, Duration::from_millis(100)); - - // Groups 3, 5, 7 — non-sequential with gaps. - // After startup selects group 3 (earliest with data), consumer reads it, - // then blocks on gap (waiting for group 4 which never arrives). - write_group(&mut track, 3, &[ts(0)]); - write_group(&mut track, 5, &[ts(150_000)]); - - // Group 7: write one frame now, push a second later to trigger the latency skip. - let mut group7 = track.create_group(moq_lite::Group { sequence: 7 }).unwrap(); - Frame { - timestamp: ts(300_000), - payload: BufList::from_iter(vec![Bytes::from_static(&[0xDE, 0xAD])]), - } - .encode(&mut group7) - .unwrap(); - - let finisher = tokio::spawn(async move { - // Wait for the consumer to process groups 3 and 5, then push - // a second frame on group 7 with a high enough timestamp to - // trigger the latency skip past the gap at group 6. - tokio::time::sleep(Duration::from_millis(50)).await; - Frame { - timestamp: ts(400_000), - payload: BufList::from_iter(vec![Bytes::from_static(&[0xBE, 0xEF])]), - } - .encode(&mut group7) - .unwrap(); - group7.finish().unwrap(); - track.finish().unwrap(); - }); - - let frames = tokio::time::timeout(Duration::from_secs(2), async { - let mut frames = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame); - } - frames - }) - .await - .expect("should not hang"); - - // Startup picks group 3 (earliest with data), reads it. - // Blocks on gap at 4. Latency skip: min(5)=150ms, max(7)=400ms → skip to 5. - // Reads group 5, blocks on gap at 6. Another skip to group 7. - assert_eq!(frames[0].group, 3); - assert_eq!(frames[1].group, 5); - assert!(frames.iter().skip(2).all(|f| f.group == 7)); - finisher.await.unwrap(); - } - - #[tokio::test] - async fn startup_skips_groups_without_data() { - tokio::time::pause(); - let mut track = moq_lite::Track::new("test").produce(); - let consumer_track = track.consume(); - let mut consumer = OrderedConsumer::new(consumer_track, Duration::from_millis(500)); - - // Group 5: no frames written yet (pending) - let _group5 = track.create_group(moq_lite::Group { sequence: 5 }).unwrap(); - // Group 7: has data - write_group(&mut track, 7, &[ts(210_000)]); - track.finish().unwrap(); - - let frames = tokio::time::timeout(Duration::from_millis(500), async { - let mut frames = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame); - } - frames - }) - .await - .expect("should not hang"); - - assert!(!frames.is_empty()); - // Group 7 should be selected since group 5 has no data. - assert_eq!(frames[0].group, 7); - } - - #[tokio::test] - async fn startup_single_group_mid_stream() { - let mut track = moq_lite::Track::new("test").produce(); - let consumer_track = track.consume(); - let mut consumer = OrderedConsumer::new(consumer_track, Duration::from_millis(500)); - - // Only group 100 exists. - write_group(&mut track, 100, &[ts(3_000_000)]); - track.finish().unwrap(); - - let frames = read_all(&mut consumer).await.unwrap(); - assert_eq!(frames.len(), 1); - assert_eq!(frames[0].group, 100); - } - - #[tokio::test] - async fn multiple_sequential_latency_skips() { - tokio::time::pause(); - let mut track = moq_lite::Track::new("test").produce(); - let consumer_track = track.consume(); - let mut consumer = OrderedConsumer::new(consumer_track, Duration::from_millis(50)); - - // Group 0: blocks - let mut group0 = track.create_group(moq_lite::Group { sequence: 0 }).unwrap(); - Frame { - timestamp: ts(0), - payload: BufList::from_iter(vec![Bytes::from_static(&[0xAA])]), - } - .encode(&mut group0) - .unwrap(); - - // Groups 1-3: each 100ms apart, triggering skips (> 50ms latency) - write_group(&mut track, 1, &[ts(100_000)]); - write_group(&mut track, 2, &[ts(200_000)]); - write_group(&mut track, 3, &[ts(300_000)]); - track.finish().unwrap(); - - let finisher = tokio::spawn(async move { - tokio::time::sleep(Duration::from_millis(20)).await; - group0.finish().unwrap(); - }); - - let frames = read_all(&mut consumer).await.unwrap(); - assert!(!frames.is_empty()); - finisher.await.unwrap(); - } - - #[tokio::test] - async fn latency_skip_boundary_exact() { - tokio::time::pause(); - let mut track = moq_lite::Track::new("test").produce(); - let consumer_track = track.consume(); - let mut consumer = OrderedConsumer::new(consumer_track, Duration::from_millis(100)); - - // Group 0: blocks - let mut group0 = track.create_group(moq_lite::Group { sequence: 0 }).unwrap(); - Frame { - timestamp: ts(0), - payload: BufList::from_iter(vec![Bytes::from_static(&[0xAA])]), - } - .encode(&mut group0) - .unwrap(); - - // Group 1: exactly 100ms span (>= max_latency should trigger skip) - write_group(&mut track, 1, &[ts(100_000)]); - track.finish().unwrap(); - - let finisher = tokio::spawn(async move { - tokio::time::sleep(Duration::from_millis(20)).await; - group0.finish().unwrap(); - }); - - let frames = read_all(&mut consumer).await.unwrap(); - assert!(!frames.is_empty()); - finisher.await.unwrap(); - } - - #[tokio::test] - async fn group_error_skips_to_next() { - let mut track = moq_lite::Track::new("test").produce(); - let consumer_track = track.consume(); - let mut consumer = OrderedConsumer::new(consumer_track, Duration::from_millis(500)); - - // Group 0: aborted - let mut group0 = track.create_group(moq_lite::Group { sequence: 0 }).unwrap(); - group0.abort(moq_lite::Error::Cancel).unwrap(); - - // Group 1: valid - write_group(&mut track, 1, &[ts(30_000)]); - track.finish().unwrap(); - - let frames = read_all(&mut consumer).await.unwrap(); - assert_eq!(frames.len(), 1); - assert_eq!(frames[0].group, 1); - } - - #[tokio::test] - async fn track_finishes_while_reading() { - tokio::time::pause(); - let mut track = moq_lite::Track::new("test").produce(); - let consumer_track = track.consume(); - let mut consumer = OrderedConsumer::new(consumer_track, Duration::from_millis(500)); - - write_group(&mut track, 0, &[ts(0)]); - - // Finish the track after a delay, simulating incremental arrival. - let finisher = tokio::spawn(async move { - tokio::time::sleep(Duration::from_millis(20)).await; - write_group(&mut track, 1, &[ts(30_000)]); - tokio::time::sleep(Duration::from_millis(20)).await; - track.finish().unwrap(); - }); - - let frames = tokio::time::timeout(Duration::from_secs(2), async { - let mut frames = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame); - } - frames - }) - .await - .expect("consumer should not hang"); - - assert_eq!(frames.len(), 2); - finisher.await.unwrap(); - } - - #[tokio::test] - async fn empty_group_advances() { - let mut track = moq_lite::Track::new("test").produce(); - let consumer_track = track.consume(); - let mut consumer = OrderedConsumer::new(consumer_track, Duration::from_millis(500)); - - // Group 0: empty (no frames, just finished) - let mut group0 = track.create_group(moq_lite::Group { sequence: 0 }).unwrap(); - group0.finish().unwrap(); - - // Group 1: has data - write_group(&mut track, 1, &[ts(30_000)]); - track.finish().unwrap(); - - let frames = read_all(&mut consumer).await.unwrap(); - assert_eq!(frames.len(), 1); - assert_eq!(frames[0].group, 1); + pub async fn read(&mut self) -> Result, crate::Error> { + panic!("hang::container::OrderedConsumer has been moved to moq_mux::consumer::OrderedConsumer") } } diff --git a/rs/hang/src/container/mod.rs b/rs/hang/src/container/mod.rs index e917ead2c..78d6ca642 100644 --- a/rs/hang/src/container/mod.rs +++ b/rs/hang/src/container/mod.rs @@ -2,6 +2,7 @@ mod consumer; mod frame; mod producer; +#[allow(deprecated)] pub use consumer::*; pub use frame::*; pub use producer::*; diff --git a/rs/hang/src/container/producer.rs b/rs/hang/src/container/producer.rs index f179e364d..0b56eb696 100644 --- a/rs/hang/src/container/producer.rs +++ b/rs/hang/src/container/producer.rs @@ -1,4 +1,4 @@ -use super::{Frame, OrderedConsumer, Timestamp}; +use super::{Frame, Timestamp}; use crate::Error; /// A producer for media tracks with group management. @@ -128,14 +128,6 @@ impl OrderedProducer { self.track.finish()?; Ok(()) } - - /// Create a consumer for this track. - /// - /// Multiple consumers can be created from the same producer, each receiving - /// a copy of all data written to the track. - pub fn consume(&self, max_latency: std::time::Duration) -> OrderedConsumer { - OrderedConsumer::new(self.track.consume(), max_latency) - } } impl From for OrderedProducer { diff --git a/rs/libmoq/src/consume.rs b/rs/libmoq/src/consume.rs index 5fbfdd6ae..d018589f4 100644 --- a/rs/libmoq/src/consume.rs +++ b/rs/libmoq/src/consume.rs @@ -5,6 +5,8 @@ use tokio::sync::oneshot; use crate::ffi::OnStatus; use crate::{Error, Id, NonZeroSlab, State, moq_audio_config, moq_frame, moq_video_config}; +type LegacyConsumer = moq_mux::consumer::OrderedConsumer; + struct ConsumeCatalog { broadcast: moq_lite::BroadcastConsumer, @@ -38,7 +40,7 @@ pub struct Consume { track_task: NonZeroSlab>, /// Buffered frames ready for consumption. - frame: NonZeroSlab, + frame: NonZeroSlab, } impl Consume { @@ -221,7 +223,7 @@ impl Consume { name: rendition.clone(), priority: 1, // TODO: Remove priority })?; - let track = hang::container::OrderedConsumer::new(track, latency); + let track = LegacyConsumer::new(track, moq_mux::consumer::Legacy, latency); let channel = oneshot::channel(); let entry = TaskEntry { @@ -265,7 +267,7 @@ impl Consume { name: rendition.clone(), priority: 2, // TODO: Remove priority })?; - let track = hang::container::OrderedConsumer::new(track, latency); + let track = LegacyConsumer::new(track, moq_mux::consumer::Legacy, latency); let channel = oneshot::channel(); let entry = TaskEntry { @@ -289,7 +291,7 @@ impl Consume { Ok(id) } - async fn run_track(task_id: Id, mut track: hang::container::OrderedConsumer) -> Result<(), Error> { + async fn run_track(task_id: Id, mut track: LegacyConsumer) -> Result<(), Error> { while let Some(mut ordered) = track.read().await? { // TODO add a chunking API so we don't have to (potentially) allocate a contiguous buffer for the frame. let mut new_payload = hang::container::BufList::new(); @@ -301,7 +303,7 @@ impl Consume { ordered.payload.copy_to_bytes(ordered.payload.num_bytes()) }); - let new_frame = hang::container::OrderedFrame { + let new_frame = moq_mux::consumer::OrderedFrame { timestamp: ordered.timestamp, payload: new_payload, group: ordered.group, diff --git a/rs/libmoq/src/error.rs b/rs/libmoq/src/error.rs index 17eddda15..fe250ffca 100644 --- a/rs/libmoq/src/error.rs +++ b/rs/libmoq/src/error.rs @@ -128,6 +128,15 @@ pub enum Error { NulError(#[from] std::ffi::NulError), } +impl From for Error { + fn from(err: moq_mux::consumer::Error) -> Self { + match err { + moq_mux::consumer::Error::Moq(e) => Error::Moq(e), + e => Error::DecodeFailed(Arc::new(e.into())), + } + } +} + impl From for Error { fn from(err: tracing::metadata::ParseLevelError) -> Self { Error::Level(Arc::new(err)) diff --git a/rs/libmoq/src/publish.rs b/rs/libmoq/src/publish.rs index 42ed94c3c..ed8a5d9e9 100644 --- a/rs/libmoq/src/publish.rs +++ b/rs/libmoq/src/publish.rs @@ -1,7 +1,7 @@ use std::{str::FromStr, sync::Arc}; use bytes::Buf; -use moq_mux::import; +use moq_mux::producer; use crate::{Error, Id, NonZeroSlab}; @@ -11,7 +11,7 @@ pub struct Publish { broadcasts: NonZeroSlab<(moq_lite::BroadcastProducer, moq_mux::CatalogProducer)>, /// Active media encoders/decoders for publishing. - media: NonZeroSlab, + media: NonZeroSlab, } impl Publish { @@ -38,8 +38,8 @@ impl Publish { pub fn media_ordered(&mut self, broadcast: Id, format: &str, mut init: &[u8]) -> Result { let (broadcast, catalog) = self.broadcasts.get(broadcast).ok_or(Error::BroadcastNotFound)?; - let format = import::DecoderFormat::from_str(format).map_err(|_| Error::UnknownFormat(format.to_string()))?; - let decoder = import::Decoder::new(broadcast.clone(), catalog.clone(), format, &mut init) + let format = producer::FramedFormat::from_str(format).map_err(|_| Error::UnknownFormat(format.to_string()))?; + let decoder = producer::Framed::new(broadcast.clone(), catalog.clone(), format, &mut init) .map_err(|err| Error::InitFailed(Arc::new(err)))?; let id = self.media.insert(decoder)?; diff --git a/rs/moq-cli/Cargo.toml b/rs/moq-cli/Cargo.toml index 7facec30e..f7cb9d7f2 100644 --- a/rs/moq-cli/Cargo.toml +++ b/rs/moq-cli/Cargo.toml @@ -23,10 +23,12 @@ websocket = ["moq-native/websocket"] anyhow = { version = "1", features = ["backtrace"] } axum = { version = "0.8", features = ["tokio"] } axum-server = { version = "0.8", features = ["tls-rustls"] } +base64 = "0.22" clap = { version = "4", features = ["derive"] } hang = { workspace = true } moq-mux = { workspace = true } moq-native = { workspace = true, default-features = false, features = ["aws-lc-rs"] } +mp4-atom = "0.10.0" rustls = { version = "0.23", features = ["aws-lc-rs"], default-features = false } tokio = { workspace = true, features = ["full"] } tower-http = { version = "0.6", features = ["cors", "fs"] } diff --git a/rs/moq-cli/src/client.rs b/rs/moq-cli/src/client.rs deleted file mode 100644 index e33d51c7c..000000000 --- a/rs/moq-cli/src/client.rs +++ /dev/null @@ -1,29 +0,0 @@ -use crate::Publish; - -use hang::moq_lite; -use url::Url; - -pub async fn run_client(client: moq_native::Client, url: Url, name: String, publish: Publish) -> anyhow::Result<()> { - // Create an origin producer to publish to the broadcast. - let origin = moq_lite::Origin::produce(); - origin.publish_broadcast(&name, publish.consume()); - - tracing::info!(%url, %name, "connecting"); - - // Establish the connection, not providing a subscriber. - let mut session = client.with_publish(origin.consume()).connect(url).await?; - - #[cfg(unix)] - // Notify systemd that we're ready. - let _ = sd_notify::notify(true, &[sd_notify::NotifyState::Ready]); - - tokio::select! { - res = publish.run() => res, - res = session.closed() => res.map_err(Into::into), - _ = tokio::signal::ctrl_c() => { - session.close(moq_lite::Error::Cancel); - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - Ok(()) - }, - } -} diff --git a/rs/moq-cli/src/main.rs b/rs/moq-cli/src/main.rs index e91ba34e7..1648dcced 100644 --- a/rs/moq-cli/src/main.rs +++ b/rs/moq-cli/src/main.rs @@ -1,14 +1,13 @@ -mod client; mod publish; -mod server; +mod subscribe; mod web; -use client::*; use publish::*; -use server::*; +use subscribe::*; use web::*; use clap::{Parser, Subcommand}; +use hang::moq_lite; use std::path::PathBuf; use url::Url; @@ -17,9 +16,8 @@ pub struct Cli { #[command(flatten)] log: moq_native::Log, - /// Iroh configuration - #[command(flatten)] #[cfg(feature = "iroh")] + #[command(flatten)] iroh: moq_native::IrohEndpointConfig, #[command(subcommand)] @@ -28,54 +26,54 @@ pub struct Cli { #[derive(Subcommand, Clone)] pub enum Command { - Serve { + /// Run as a server (listen for connections) + Server { #[command(flatten)] config: moq_native::ServerConfig, - /// The name of the broadcast to serve. - #[arg(long)] - name: String, - /// Optionally serve static files from the given directory. #[arg(long)] dir: Option, - /// The format of the input media. #[command(subcommand)] - format: PublishFormat, + action: Action, }, - Publish { - /// The MoQ client configuration. + /// Run as a client (connect to a server) + Client { #[command(flatten)] config: moq_native::ClientConfig, /// The URL of the MoQ server. - /// - /// The URL must start with `https://` or `http://`. - /// - If `http` is used, a HTTP fetch to "/certificate.sha256" is first made to get the TLS certificiate fingerprint (insecure). - /// - If `https` is used, then A WebTransport connection is made via QUIC to the provided host/port. - /// - /// The `?jwt=` query parameter is used to provide a JWT token from moq-token-cli. - /// Otherwise, the public path (if any) is used instead. - /// - /// The path currently must be `/` or you'll get an error on connect. #[arg(long)] url: Url, - /// The name of the broadcast to publish. + #[command(subcommand)] + action: Action, + }, +} + +#[derive(Subcommand, Clone)] +pub enum Action { + /// Publish media from stdin + Publish { #[arg(long)] name: String, - /// The format of the input media. - #[command(subcommand)] - format: PublishFormat, + #[command(flatten)] + args: PublishArgs, + }, + /// Subscribe to media, write to stdout + Subscribe { + #[arg(long)] + name: String, + + #[command(flatten)] + args: SubscribeArgs, }, } #[tokio::main] async fn main() -> anyhow::Result<()> { - // TODO: It would be nice to remove this and rely on feature flags only. - // However, some dependency is pulling in `ring` and I don't know why, so meh for now. rustls::crypto::aws_lc_rs::default_provider() .install_default() .expect("failed to install default crypto provider"); @@ -83,16 +81,11 @@ async fn main() -> anyhow::Result<()> { let cli = Cli::parse(); cli.log.init()?; - let publish = Publish::new(match &cli.command { - Command::Serve { format, .. } => format, - Command::Publish { format, .. } => format, - })?; - #[cfg(feature = "iroh")] let iroh = cli.iroh.bind().await?; match cli.command { - Command::Serve { config, dir, name, .. } => { + Command::Server { config, dir, action } => { let web_bind = config.bind.unwrap_or("[::]:443".parse().unwrap()); let server = config.init()?; @@ -101,19 +94,169 @@ async fn main() -> anyhow::Result<()> { let web_tls = server.tls_info(); - tokio::select! { - res = run_server(server, name, publish.consume()) => res, - res = run_web(web_bind, web_tls, dir) => res, - res = publish.run() => res, + match action { + Action::Publish { name, args } => { + let publish = Publish::new(&args)?; + let broadcast = publish.consume(); + + tokio::select! { + res = run_publish_server(server, name, broadcast) => res, + res = publish.run() => res, + res = run_web(web_bind, web_tls, dir) => res, + } + } + Action::Subscribe { name, args } => { + let origin = moq_lite::Origin::produce(); + let mut consumer = origin.consume(); + let server = server.with_consume(origin); + + tokio::select! { + res = run_accept(server) => res, + res = async { + let broadcast = wait_broadcast(&mut consumer, &name).await?; + Subscribe::new(broadcast, args).run().await + } => res, + res = run_web(web_bind, web_tls, dir) => res, + } + } } } - Command::Publish { config, url, name, .. } => { + Command::Client { config, url, action } => { let client = config.init()?; #[cfg(feature = "iroh")] let client = client.with_iroh(iroh); - run_client(client, url, name, publish).await + match action { + Action::Publish { name, args } => { + let publish = Publish::new(&args)?; + let origin = moq_lite::Origin::produce(); + origin.publish_broadcast(&name, publish.consume()); + + tracing::info!(%url, %name, "connecting"); + let session = client.with_publish(origin.consume()).connect(url).await?; + + run_client(session, publish.run()).await + } + Action::Subscribe { name, args } => { + let origin = moq_lite::Origin::produce(); + let mut consumer = origin.consume(); + + tracing::info!(%url, %name, "connecting"); + let session = client.with_consume(origin).connect(url).await?; + + let broadcast = wait_broadcast(&mut consumer, &name).await?; + let subscribe = Subscribe::new(broadcast, args); + + run_client(session, subscribe.run()).await + } + } + } + } +} + +/// Run a client session, waiting for the action to complete, the session to close, or ctrl_c. +async fn run_client( + mut session: moq_lite::Session, + action: impl std::future::Future>, +) -> anyhow::Result<()> { + #[cfg(unix)] + let _ = sd_notify::notify(true, &[sd_notify::NotifyState::Ready]); + + tokio::select! { + res = action => res, + res = session.closed() => res.map_err(Into::into), + _ = tokio::signal::ctrl_c() => { + session.close(moq_lite::Error::Cancel); + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + Ok(()) + }, + } +} + +/// Wait for a named broadcast to be announced on the origin. +async fn wait_broadcast( + consumer: &mut moq_lite::OriginConsumer, + name: &str, +) -> anyhow::Result { + loop { + let (path, announced) = consumer + .announced() + .await + .ok_or_else(|| anyhow::anyhow!("origin closed"))?; + + if let Some(broadcast) = announced { + if path.as_ref() == name { + return Ok(broadcast); + } } } } + +/// Accept connections in a loop, publishing the same broadcast to each. +async fn run_publish_server( + mut server: moq_native::Server, + name: String, + broadcast: moq_lite::BroadcastConsumer, +) -> anyhow::Result<()> { + #[cfg(unix)] + let _ = sd_notify::notify(true, &[sd_notify::NotifyState::Ready]); + + let mut conn_id: u64 = 0; + + tracing::info!(addr = ?server.local_addr(), "listening"); + + while let Some(request) = server.accept().await { + let id = conn_id; + conn_id += 1; + + let name = name.clone(); + let broadcast = broadcast.clone(); + + tokio::spawn(async move { + let origin = moq_lite::Origin::produce(); + origin.publish_broadcast(&name, broadcast); + + match request.with_publish(origin.consume()).ok().await { + Ok(session) => { + tracing::info!(id, "accepted session"); + if let Err(err) = session.closed().await { + tracing::warn!(id, %err, "session error"); + } + } + Err(err) => tracing::warn!(id, %err, "failed to accept session"), + } + }); + } + + Ok(()) +} + +/// Accept connections in a loop (origin already configured on the server). +async fn run_accept(mut server: moq_native::Server) -> anyhow::Result<()> { + #[cfg(unix)] + let _ = sd_notify::notify(true, &[sd_notify::NotifyState::Ready]); + + let mut conn_id: u64 = 0; + + tracing::info!(addr = ?server.local_addr(), "listening"); + + while let Some(request) = server.accept().await { + let id = conn_id; + conn_id += 1; + + tokio::spawn(async move { + match request.ok().await { + Ok(session) => { + tracing::info!(id, "accepted session"); + if let Err(err) = session.closed().await { + tracing::warn!(id, %err, "session error"); + } + } + Err(err) => tracing::warn!(id, %err, "failed to accept session"), + } + }); + } + + Ok(()) +} diff --git a/rs/moq-cli/src/publish.rs b/rs/moq-cli/src/publish.rs index 3580eb558..ffc8c5c7b 100644 --- a/rs/moq-cli/src/publish.rs +++ b/rs/moq-cli/src/publish.rs @@ -1,88 +1,143 @@ -use clap::Subcommand; +use clap::ValueEnum; use hang::moq_lite; -use moq_mux::import; +use moq_mux::producer; -#[derive(Subcommand, Clone)] -pub enum PublishFormat { +#[derive(ValueEnum, Clone, Copy)] +pub enum InputFormat { + Fmp4, Avc3, - Fmp4 { - /// Transmit the fMP4 container directly instead of decoding it. - #[arg(long)] - passthrough: bool, - }, - // NOTE: No aac support because it needs framing. - Hls { - /// URL or file path of an HLS playlist to ingest. - #[arg(long)] - playlist: String, - - /// Transmit the fMP4 segments directly instead of decoding them. - #[arg(long)] - passthrough: bool, - }, + Hls, } -enum PublishDecoder { - Avc3(Box), - Fmp4(Box), - Hls(Box), +#[derive(ValueEnum, Clone, Copy)] +pub enum ExportFormat { + Hang, + Fmp4, +} + +#[derive(clap::Args, Clone)] +pub struct PublishArgs { + /// Input format (what's being read from stdin). + /// For hls, provide the playlist URL on stdin. + #[arg(long)] + pub input: InputFormat, + + /// Convert to a different format before publishing. + /// If not specified, publishes in the import's native format. + #[arg(long)] + pub export: Option, +} + +enum PublishKind { + Avc3(Box), + Fmp4(Box), + Hls(Box), } pub struct Publish { - decoder: PublishDecoder, - broadcast: moq_lite::BroadcastProducer, + kind: PublishKind, + export: Option, + + /// The broadcast the importer writes into. + import_broadcast: moq_lite::BroadcastProducer, + + /// The broadcast that gets published (after optional conversion). + output_broadcast: moq_lite::BroadcastProducer, } impl Publish { - pub fn new(format: &PublishFormat) -> anyhow::Result { - let mut broadcast = moq_lite::Broadcast::new().produce(); - let catalog = moq_mux::CatalogProducer::new(&mut broadcast)?; - - let decoder = match format { - PublishFormat::Avc3 => { - let avc3 = import::Avc3::new(broadcast.clone(), catalog.clone()); - PublishDecoder::Avc3(Box::new(avc3)) + pub fn new(args: &PublishArgs) -> anyhow::Result { + let mut import_broadcast = moq_lite::Broadcast::new().produce(); + let catalog = moq_mux::CatalogProducer::new(&mut import_broadcast)?; + + let kind = match args.input { + InputFormat::Avc3 => { + let avc3 = producer::Avc3::new(import_broadcast.clone(), catalog.clone()); + PublishKind::Avc3(Box::new(avc3)) } - PublishFormat::Fmp4 { passthrough } => { - let fmp4 = import::Fmp4::new( - broadcast.clone(), - catalog.clone(), - import::Fmp4Config { - passthrough: *passthrough, - }, - ); - PublishDecoder::Fmp4(Box::new(fmp4)) + InputFormat::Fmp4 => { + let fmp4 = producer::Fmp4::new(import_broadcast.clone(), catalog.clone()); + PublishKind::Fmp4(Box::new(fmp4)) } - PublishFormat::Hls { playlist, passthrough } => { - let hls = import::Hls::new( - broadcast.clone(), - catalog.clone(), - import::HlsConfig { - playlist: playlist.clone(), - client: None, - passthrough: *passthrough, - }, - )?; - PublishDecoder::Hls(Box::new(hls)) + InputFormat::Hls => { + // Read playlist URL from stdin (first line) + let mut playlist = String::new(); + std::io::stdin().read_line(&mut playlist)?; + let playlist = playlist.trim().to_string(); + anyhow::ensure!(!playlist.is_empty(), "expected HLS playlist URL on stdin"); + + let config = producer::HlsConfig::new(playlist); + let hls = producer::Hls::new(import_broadcast.clone(), catalog.clone(), config)?; + PublishKind::Hls(Box::new(hls)) } }; - Ok(Self { decoder, broadcast }) + // If exporting, create a separate output broadcast for the converter. + // Otherwise, the output is the same as the import. + let output_broadcast = if args.export.is_some() { + moq_lite::Broadcast::new().produce() + } else { + import_broadcast.clone() + }; + + Ok(Self { + kind, + export: args.export, + import_broadcast, + output_broadcast, + }) } pub fn consume(&self) -> moq_lite::BroadcastConsumer { - self.broadcast.consume() + self.output_broadcast.consume() } -} -impl Publish { - pub async fn run(mut self) -> anyhow::Result<()> { - let mut stdin = tokio::io::stdin(); + pub async fn run(self) -> anyhow::Result<()> { + let Self { + mut kind, + export, + import_broadcast, + output_broadcast, + } = self; + + let Some(export) = export else { + return run_import(&mut kind).await; + }; + + let import_consumer = import_broadcast.consume(); + + match export { + ExportFormat::Fmp4 => { + let converter = moq_mux::convert::Fmp4::new(import_consumer, output_broadcast); + tokio::select! { + res = run_import(&mut kind) => res, + res = converter.run() => res, + } + } + ExportFormat::Hang => { + let converter = moq_mux::convert::Hang::new(import_consumer, output_broadcast); + tokio::select! { + res = run_import(&mut kind) => res, + res = converter.run() => res, + } + } + } + } +} - match &mut self.decoder { - PublishDecoder::Avc3(decoder) => decoder.decode_from(&mut stdin).await, - PublishDecoder::Fmp4(decoder) => decoder.decode_from(&mut stdin).await, - PublishDecoder::Hls(decoder) => decoder.run().await, +async fn run_import(kind: &mut PublishKind) -> anyhow::Result<()> { + match kind { + PublishKind::Avc3(decoder) => { + let mut stdin = tokio::io::stdin(); + decoder.decode_from(&mut stdin).await + } + PublishKind::Fmp4(decoder) => { + let mut stdin = tokio::io::stdin(); + decoder.decode_from(&mut stdin).await + } + PublishKind::Hls(hls) => { + hls.init().await?; + hls.run().await } } } diff --git a/rs/moq-cli/src/server.rs b/rs/moq-cli/src/server.rs deleted file mode 100644 index eaed96dd6..000000000 --- a/rs/moq-cli/src/server.rs +++ /dev/null @@ -1,51 +0,0 @@ -use hang::moq_lite; - -pub async fn run_server( - mut server: moq_native::Server, - name: String, - consumer: moq_lite::BroadcastConsumer, -) -> anyhow::Result<()> { - #[cfg(unix)] - // Notify systemd that we're ready. - let _ = sd_notify::notify(true, &[sd_notify::NotifyState::Ready]); - - let mut conn_id = 0; - - tracing::info!(addr = ?server.local_addr(), "listening"); - - while let Some(session) = server.accept().await { - let id = conn_id; - conn_id += 1; - - let name = name.clone(); - - let consumer = consumer.clone(); - // Handle the connection in a new task. - tokio::spawn(async move { - if let Err(err) = run_session(id, session, name, consumer).await { - tracing::warn!(%err, "failed to accept session"); - } - }); - } - - Ok(()) -} - -#[tracing::instrument("session", skip_all, fields(id))] -async fn run_session( - id: u64, - session: moq_native::Request, - name: String, - consumer: moq_lite::BroadcastConsumer, -) -> anyhow::Result<()> { - // Create an origin producer to publish to the broadcast. - let origin = moq_lite::Origin::produce(); - origin.publish_broadcast(&name, consumer); - - // Blindly accept the session (WebTransport or QUIC), regardless of the URL. - let session = session.with_publish(origin.consume()).ok().await?; - - tracing::info!(id, "accepted session"); - - session.closed().await.map_err(Into::into) -} diff --git a/rs/moq-cli/src/subscribe.rs b/rs/moq-cli/src/subscribe.rs new file mode 100644 index 000000000..200549d58 --- /dev/null +++ b/rs/moq-cli/src/subscribe.rs @@ -0,0 +1,146 @@ +use anyhow::Context; +use clap::ValueEnum; +use hang::moq_lite; +use tokio::io::AsyncWriteExt; + +#[derive(ValueEnum, Clone, Copy)] +pub enum OutputFormat { + Fmp4, +} + +#[derive(clap::Args, Clone)] +pub struct SubscribeArgs { + /// Output format for stdout + #[arg(long)] + pub output: OutputFormat, + + /// Maximum latency in milliseconds before skipping groups + #[arg(long, default_value = "500")] + pub max_latency: u64, +} + +pub struct Subscribe { + broadcast: moq_lite::BroadcastConsumer, + args: SubscribeArgs, +} + +impl Subscribe { + pub fn new(broadcast: moq_lite::BroadcastConsumer, args: SubscribeArgs) -> Self { + Self { broadcast, args } + } + + pub async fn run(self) -> anyhow::Result<()> { + match self.args.output { + OutputFormat::Fmp4 => self.run_fmp4().await, + } + } + + async fn run_fmp4(self) -> anyhow::Result<()> { + // Always convert to CMAF — this is a no-op for tracks already in CMAF. + let cmaf_output = moq_lite::Broadcast::new().produce(); + let cmaf_consumer = cmaf_output.consume(); + let converter = moq_mux::convert::Fmp4::new(self.broadcast, cmaf_output); + + // Subscribe to the catalog before the converter starts, so we don't miss it. + let catalog_track = cmaf_consumer.subscribe_track(&hang::Catalog::default_track())?; + + let max_latency = std::time::Duration::from_millis(self.args.max_latency); + + // Run the converter concurrently — it blocks until all tracks finish, + // so we must read from the output broadcast in parallel. + tokio::select! { + res = converter.run() => res?, + res = mux_fmp4(catalog_track, cmaf_consumer, max_latency) => res?, + } + + Ok(()) + } +} + +async fn mux_fmp4( + catalog_track: moq_lite::TrackConsumer, + cmaf_consumer: moq_lite::BroadcastConsumer, + max_latency: std::time::Duration, +) -> anyhow::Result<()> { + let mut stdout = tokio::io::stdout(); + + let mut catalog_consumer = hang::CatalogConsumer::new(catalog_track); + let catalog = catalog_consumer.next().await?.context("empty catalog")?; + + // Build exporter from catalog (for init segment) + let exporter = moq_mux::consumer::Fmp4::new(&catalog)?; + + // Write init segment (merged multi-track moov) + let init = exporter.init(&catalog)?; + stdout.write_all(&init).await?; + stdout.flush().await?; + + // Build OrderedMuxer from all track consumers (all CMAF after conversion) + let mut muxer_tracks = Vec::new(); + + for (name, config) in &catalog.video.renditions { + let track = cmaf_consumer.subscribe_track(&moq_lite::Track { + name: name.clone(), + priority: 1, + })?; + + let timescale = match &config.container { + hang::catalog::Container::Cmaf { init_data } => parse_timescale_from_init(init_data)?, + hang::catalog::Container::Legacy => { + anyhow::bail!("unexpected Legacy track after conversion") + } + }; + + let consumer = + moq_mux::consumer::OrderedConsumer::new(track, moq_mux::consumer::Cmaf { timescale }, max_latency); + muxer_tracks.push((name.clone(), consumer)); + } + + for (name, config) in &catalog.audio.renditions { + let track = cmaf_consumer.subscribe_track(&moq_lite::Track { + name: name.clone(), + priority: 2, + })?; + + let timescale = match &config.container { + hang::catalog::Container::Cmaf { init_data } => parse_timescale_from_init(init_data)?, + hang::catalog::Container::Legacy => { + anyhow::bail!("unexpected Legacy track after conversion") + } + }; + + let consumer = + moq_mux::consumer::OrderedConsumer::new(track, moq_mux::consumer::Cmaf { timescale }, max_latency); + muxer_tracks.push((name.clone(), consumer)); + } + + // Use OrderedMuxer for timestamp-ordered multi-track merge + let mut muxer = moq_mux::consumer::OrderedMuxer::new(muxer_tracks); + + while let Some(muxed) = muxer.read().await? { + // CMAF passthrough: payload is already moof+mdat + for chunk in &muxed.frame.payload { + stdout.write_all(chunk).await?; + } + stdout.flush().await?; + } + + Ok(()) +} + +fn parse_timescale_from_init(init_data_b64: &str) -> anyhow::Result { + use base64::Engine; + use mp4_atom::DecodeMaybe; + + let data = base64::engine::general_purpose::STANDARD + .decode(init_data_b64) + .context("invalid base64")?; + let mut cursor = std::io::Cursor::new(&data); + while let Some(atom) = mp4_atom::Any::decode_maybe(&mut cursor)? { + if let mp4_atom::Any::Moov(moov) = atom { + let trak = moov.trak.first().context("no tracks in moov")?; + return Ok(trak.mdia.mdhd.timescale as u64); + } + } + anyhow::bail!("no moov in init data") +} diff --git a/rs/moq-ffi/src/consumer.rs b/rs/moq-ffi/src/consumer.rs index 9ada59c5c..06ef9cfaa 100644 --- a/rs/moq-ffi/src/consumer.rs +++ b/rs/moq-ffi/src/consumer.rs @@ -6,6 +6,8 @@ use crate::error::MoqError; use crate::ffi::Task; use crate::media::*; +type LegacyConsumer = moq_mux::consumer::OrderedConsumer; + #[derive(Clone, uniffi::Object)] pub struct MoqBroadcastConsumer { inner: moq_lite::BroadcastConsumer, @@ -42,12 +44,12 @@ pub struct MoqMediaConsumer { } struct Media { - inner: hang::container::OrderedConsumer, + inner: LegacyConsumer, } impl Media { async fn next(&mut self) -> Result, MoqError> { - let Some(frame) = self.inner.read().await? else { + let Some(frame) = self.inner.read().await.map_err(|e| MoqError::Codec(e.to_string()))? else { return Ok(None); }; @@ -90,7 +92,7 @@ impl MoqBroadcastConsumer { let _guard = crate::ffi::RUNTIME.enter(); let track = self.inner.subscribe_track(&moq_lite::Track { name, priority: 0 })?; let latency = std::time::Duration::from_millis(max_latency_ms); - let consumer = hang::container::OrderedConsumer::new(track, latency); + let consumer = LegacyConsumer::new(track, moq_mux::consumer::Legacy, latency); Ok(Arc::new(MoqMediaConsumer { task: Task::new(Media { inner: consumer }), })) diff --git a/rs/moq-ffi/src/producer.rs b/rs/moq-ffi/src/producer.rs index 29470fad7..7c6d5261b 100644 --- a/rs/moq-ffi/src/producer.rs +++ b/rs/moq-ffi/src/producer.rs @@ -27,7 +27,7 @@ impl MoqBroadcastProducer { #[derive(uniffi::Object)] pub struct MoqMediaProducer { - inner: std::sync::Mutex>, + inner: std::sync::Mutex>, } #[uniffi::export] @@ -52,11 +52,11 @@ impl MoqBroadcastProducer { let _guard = crate::ffi::RUNTIME.enter(); let guard = self.state.lock().unwrap(); let state = guard.as_ref().ok_or_else(|| MoqError::Closed)?; - let format = moq_mux::import::DecoderFormat::from_str(&format) + let format = moq_mux::producer::FramedFormat::from_str(&format) .map_err(|_| MoqError::Codec(format!("unknown format: {format}")))?; let mut buf = init.as_slice(); - let decoder = moq_mux::import::Decoder::new(state.broadcast.clone(), state.catalog.clone(), format, &mut buf) + let decoder = moq_mux::producer::Framed::new(state.broadcast.clone(), state.catalog.clone(), format, &mut buf) .map_err(|err| MoqError::Codec(format!("init failed: {err}")))?; if buf.has_remaining() { diff --git a/rs/moq-mux/Cargo.toml b/rs/moq-mux/Cargo.toml index 3e847d42f..7a4b63483 100644 --- a/rs/moq-mux/Cargo.toml +++ b/rs/moq-mux/Cargo.toml @@ -29,6 +29,7 @@ anyhow = "1" base64 = "0.22" buf-list = "1" bytes = "1" +conducer = { workspace = true } h264-parser = { version = "0.4.0", optional = true } hang = { workspace = true } m3u8-rs = { version = "5", optional = true } @@ -39,6 +40,7 @@ num_enum = "0.7" reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "gzip"], optional = true } scuffle-av1 = { version = "0.1.4", optional = true } scuffle-h265 = { version = "0.2.2", optional = true } +thiserror = "2" tokio = { workspace = true, features = ["macros", "fs"] } tracing = "0.1" url = "2" @@ -46,3 +48,6 @@ url = "2" [dependencies.derive_more] version = "2" features = ["from", "display", "debug"] + +[dev-dependencies] +tokio = { workspace = true, features = ["test-util"] } diff --git a/rs/moq-mux/src/consumer/container.rs b/rs/moq-mux/src/consumer/container.rs new file mode 100644 index 000000000..f3ebf00f1 --- /dev/null +++ b/rs/moq-mux/src/consumer/container.rs @@ -0,0 +1,103 @@ +use buf_list::BufList; + +pub type Timestamp = moq_lite::Timescale<1_000_000>; + +/// Trait for parsing container-formatted frame data. +/// +/// Different container formats encode timestamps differently: +/// - Legacy (hang): VarInt timestamp prefix, stripped from payload +/// - CMAF: timestamp in moof tfdt, payload passed through unchanged +pub trait ContainerFormat { + type Error: Into; + + /// Parse timestamp from raw frame data. + /// + /// Returns (timestamp, payload) where payload may have the timestamp + /// stripped (Legacy) or be the full original data (CMAF passthrough). + fn parse(&self, payload: BufList) -> Result<(Timestamp, BufList), Self::Error>; +} + +/// Errors from container format operations. +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("moq: {0}")] + Moq(#[from] moq_lite::Error), + + #[cfg(feature = "mp4")] + #[error("mp4: {0}")] + Mp4(#[from] mp4_atom::Error), + + #[error("{0}")] + Other(String), +} + +/// hang Legacy format: VarInt timestamp prefix. +pub struct Legacy; + +impl ContainerFormat for Legacy { + type Error = moq_lite::Error; + + fn parse(&self, mut payload: BufList) -> Result<(Timestamp, BufList), Self::Error> { + let timestamp = Timestamp::decode(&mut payload)?; + Ok((timestamp, payload)) + } +} + +/// CMAF format: parse moof tfdt for timestamp, return full moof+mdat unchanged. +#[cfg(feature = "mp4")] +pub struct Cmaf { + pub timescale: u64, +} + +#[cfg(feature = "mp4")] +#[derive(Debug, thiserror::Error)] +pub enum CmafError { + #[error("mp4: {0}")] + Mp4(#[from] mp4_atom::Error), + + #[error("timestamp overflow")] + TimestampOverflow(#[from] moq_lite::TimeOverflow), + + #[error("no traf in moof")] + NoTraf, + + #[error("no tfdt in traf")] + NoTfdt, + + #[error("no moof found in CMAF frame data")] + NoMoof, +} + +#[cfg(feature = "mp4")] +impl From for Error { + fn from(e: CmafError) -> Self { + match e { + CmafError::Mp4(e) => Error::Mp4(e), + e => Error::Other(e.to_string()), + } + } +} + +#[cfg(feature = "mp4")] +impl ContainerFormat for Cmaf { + type Error = CmafError; + + fn parse(&self, payload: BufList) -> Result<(Timestamp, BufList), Self::Error> { + use mp4_atom::DecodeMaybe; + + // Collect payload into contiguous bytes for parsing + let data: Vec = payload.iter().flat_map(|c| c.iter().copied()).collect(); + let mut cursor = std::io::Cursor::new(&data); + + while let Some(atom) = mp4_atom::Any::decode_maybe(&mut cursor)? { + if let mp4_atom::Any::Moof(moof) = atom { + let traf = moof.traf.first().ok_or(CmafError::NoTraf)?; + let tfdt = traf.tfdt.as_ref().ok_or(CmafError::NoTfdt)?; + let timestamp = Timestamp::from_scale(tfdt.base_media_decode_time, self.timescale)?; + return Ok((timestamp, BufList::from_iter(vec![bytes::Bytes::from(data)]))); + } + } + + Err(CmafError::NoMoof) + } +} diff --git a/rs/moq-mux/src/consumer/fmp4.rs b/rs/moq-mux/src/consumer/fmp4.rs new file mode 100644 index 000000000..aca963278 --- /dev/null +++ b/rs/moq-mux/src/consumer/fmp4.rs @@ -0,0 +1,223 @@ +use anyhow::Context; +use base64::Engine; +use bytes::Bytes; +use hang::catalog::{Catalog, Container, VideoConfig}; +use mp4_atom::{DecodeMaybe, Encode}; + +use super::OrderedFrame; + +/// Produces fMP4 init segments and per-frame moof+mdat fragments from catalog info. +/// +/// Used for exporting a broadcast to stdout as a playable fMP4 stream. +pub struct Fmp4 { + tracks: Vec, +} + +struct Fmp4ExportTrack { + name: String, + track_id: u32, + timescale: u64, + sequence_number: u32, +} + +impl Fmp4 { + /// Build from catalog configuration. + pub fn new(catalog: &Catalog) -> anyhow::Result { + let mut tracks = Vec::new(); + let mut track_id = 1u32; + + for (name, config) in &catalog.video.renditions { + let timescale = match &config.container { + Container::Cmaf { init_data } => parse_timescale_from_init(init_data)?, + Container::Legacy => guess_video_timescale(config), + }; + + tracks.push(Fmp4ExportTrack { + name: name.clone(), + track_id, + timescale, + sequence_number: 1, + }); + track_id += 1; + } + + for (name, config) in &catalog.audio.renditions { + let timescale = match &config.container { + Container::Cmaf { init_data } => parse_timescale_from_init(init_data)?, + Container::Legacy => config.sample_rate as u64, + }; + + tracks.push(Fmp4ExportTrack { + name: name.clone(), + track_id, + timescale, + sequence_number: 1, + }); + track_id += 1; + } + + Ok(Self { tracks }) + } + + /// Generate the init segment (ftyp + moov) for all tracks. + /// + /// For multi-track output, decodes each track's init_data, extracts trak+trex, + /// and builds a merged ftyp+moov with renumbered track IDs. + pub fn init(&self, catalog: &Catalog) -> anyhow::Result { + let mut traks = Vec::new(); + let mut trexs = Vec::new(); + let mut ftyp_data = None; + + // Collect all track init data + let mut track_inits: Vec<&str> = Vec::new(); + for config in catalog.video.renditions.values() { + match &config.container { + Container::Cmaf { init_data } => track_inits.push(init_data), + Container::Legacy => anyhow::bail!("track is not CMAF"), + } + } + for config in catalog.audio.renditions.values() { + match &config.container { + Container::Cmaf { init_data } => track_inits.push(init_data), + Container::Legacy => anyhow::bail!("track is not CMAF"), + } + } + + for init_data_b64 in &track_inits { + let data = base64::engine::general_purpose::STANDARD + .decode(init_data_b64) + .context("invalid base64 init_data")?; + + let mut cursor = std::io::Cursor::new(&data); + while let Some(atom) = mp4_atom::Any::decode_maybe(&mut cursor)? { + match atom { + mp4_atom::Any::Ftyp(f) => { + if ftyp_data.is_none() { + ftyp_data = Some(f); + } + } + mp4_atom::Any::Moov(moov) => { + // Preserve original track IDs to match CMAF passthrough fragments + for trak in moov.trak { + traks.push(trak); + } + + if let Some(mvex) = moov.mvex { + for trex in mvex.trex { + trexs.push(trex); + } + } + } + _ => {} + } + } + } + + let ftyp = ftyp_data.context("no ftyp found in any init segment")?; + + let timescale = traks.first().map(|t| t.mdia.mdhd.timescale).unwrap_or(90000); + + let moov = mp4_atom::Moov { + mvhd: mp4_atom::Mvhd { + timescale, + ..Default::default() + }, + trak: traks, + mvex: if trexs.is_empty() { + None + } else { + Some(mp4_atom::Mvex { + trex: trexs, + ..Default::default() + }) + }, + ..Default::default() + }; + + let mut buf = Vec::new(); + ftyp.encode(&mut buf)?; + moov.encode(&mut buf)?; + Ok(Bytes::from(buf)) + } + + /// Encode a single frame as a moof+mdat fragment. + pub fn frame(&mut self, track_name: &str, frame: &OrderedFrame) -> anyhow::Result { + let track = self + .tracks + .iter_mut() + .find(|t| t.name == track_name) + .context("unknown track")?; + + let dts = frame.timestamp.as_micros() as u64 * track.timescale / 1_000_000; + let payload: Vec = frame.payload.clone().into_iter().flat_map(|c| c.into_iter()).collect(); + let keyframe = frame.is_keyframe(); + + let flags = if keyframe { 0x0200_0000 } else { 0x0001_0000 }; + + let seq = track.sequence_number; + track.sequence_number += 1; + + // First pass to get moof size (use Some(0) so trun includes the data_offset field) + let moof = build_moof(seq, track.track_id, dts, payload.len() as u32, flags, Some(0)); + let mut buf = Vec::new(); + moof.encode(&mut buf)?; + let moof_size = buf.len(); + + // Second pass with data_offset + let data_offset = (moof_size + 8) as i32; + let moof = build_moof(seq, track.track_id, dts, payload.len() as u32, flags, Some(data_offset)); + buf.clear(); + moof.encode(&mut buf)?; + + let mdat = mp4_atom::Mdat { data: payload }; + mdat.encode(&mut buf)?; + + Ok(Bytes::from(buf)) + } +} + +fn build_moof(seq: u32, track_id: u32, dts: u64, size: u32, flags: u32, data_offset: Option) -> mp4_atom::Moof { + mp4_atom::Moof { + mfhd: mp4_atom::Mfhd { sequence_number: seq }, + traf: vec![mp4_atom::Traf { + tfhd: mp4_atom::Tfhd { + track_id, + ..Default::default() + }, + tfdt: Some(mp4_atom::Tfdt { + base_media_decode_time: dts, + }), + trun: vec![mp4_atom::Trun { + data_offset, + entries: vec![mp4_atom::TrunEntry { + size: Some(size), + flags: Some(flags), + ..Default::default() + }], + }], + ..Default::default() + }], + } +} + +fn parse_timescale_from_init(init_data_b64: &str) -> anyhow::Result { + let data = base64::engine::general_purpose::STANDARD + .decode(init_data_b64) + .context("invalid base64")?; + let mut cursor = std::io::Cursor::new(&data); + while let Some(atom) = mp4_atom::Any::decode_maybe(&mut cursor)? { + if let mp4_atom::Any::Moov(moov) = atom { + let trak = moov.trak.first().context("no tracks in moov")?; + return Ok(trak.mdia.mdhd.timescale as u64); + } + } + anyhow::bail!("no moov in init data") +} + +fn guess_video_timescale(config: &VideoConfig) -> u64 { + if let Some(fps) = config.framerate { + (fps * 1000.0) as u64 + } else { + 90000 + } +} diff --git a/rs/moq-mux/src/consumer/frame.rs b/rs/moq-mux/src/consumer/frame.rs new file mode 100644 index 000000000..4789054bc --- /dev/null +++ b/rs/moq-mux/src/consumer/frame.rs @@ -0,0 +1,29 @@ +use buf_list::BufList; + +use super::Timestamp; + +/// A frame returned by [`super::OrderedConsumer::read()`] with group context. +#[derive(Clone, Debug)] +pub struct OrderedFrame { + /// The presentation timestamp for this frame. + pub timestamp: Timestamp, + + /// The encoded media data for this frame, split into chunks. + pub payload: BufList, + + /// The group sequence number this frame belongs to. + pub group: u64, + + /// The frame index within the group (0 = first frame in the group). + /// + /// With duration-based grouping (e.g. audio), the first frame is not + /// necessarily a keyframe — it only denotes position within the group. + pub index: usize, +} + +impl OrderedFrame { + /// Returns true if this is the first frame in the group (index 0). + pub fn is_keyframe(&self) -> bool { + self.index == 0 + } +} diff --git a/rs/moq-mux/src/consumer/mod.rs b/rs/moq-mux/src/consumer/mod.rs new file mode 100644 index 000000000..f119603f0 --- /dev/null +++ b/rs/moq-mux/src/consumer/mod.rs @@ -0,0 +1,11 @@ +mod container; +mod fmp4; +mod frame; +mod muxer; +mod ordered; + +pub use container::*; +pub use fmp4::*; +pub use frame::*; +pub use muxer::*; +pub use ordered::*; diff --git a/rs/moq-mux/src/consumer/muxer.rs b/rs/moq-mux/src/consumer/muxer.rs new file mode 100644 index 000000000..b9b072422 --- /dev/null +++ b/rs/moq-mux/src/consumer/muxer.rs @@ -0,0 +1,103 @@ +use std::task::Poll; + +use super::container::Error; +use super::{ContainerFormat, OrderedConsumer, OrderedFrame}; + +/// A frame returned by [`OrderedMuxer::read()`] with its track name. +pub struct MuxedFrame { + /// The track name this frame belongs to. + pub name: String, + /// The frame data. + pub frame: OrderedFrame, +} + +/// Merges multiple track consumers into a single timestamp-ordered stream. +/// +/// Given N consumers (one per track), yields frames in ascending timestamp order +/// across all tracks. This enables proper interleaving for multi-track fMP4 output. +pub struct OrderedMuxer { + tracks: Vec>, +} + +struct MuxerTrack { + name: String, + consumer: OrderedConsumer, + pending: Option, + finished: bool, +} + +impl OrderedMuxer { + /// Create a new muxer from a list of (name, consumer) pairs. + pub fn new(tracks: Vec<(String, OrderedConsumer)>) -> Self { + Self { + tracks: tracks + .into_iter() + .map(|(name, consumer)| MuxerTrack { + name, + consumer, + pending: None, + finished: false, + }) + .collect(), + } + } + + /// Read the next frame in timestamp order across all tracks. + /// + /// Returns `None` when all tracks have ended. + pub async fn read(&mut self) -> Result, Error> { + conducer::wait(|waiter| self.poll_read(waiter)).await + } + + /// Poll-based implementation. + pub fn poll_read(&mut self, waiter: &conducer::Waiter) -> Poll, Error>> { + // Fill empty pending slots + for track in &mut self.tracks { + if track.pending.is_none() && !track.finished { + match track.consumer.poll_read(waiter) { + Poll::Ready(Ok(Some(frame))) => { + track.pending = Some(frame); + } + Poll::Ready(Ok(None)) => { + track.finished = true; + } + Poll::Ready(Err(e)) => { + track.finished = true; + tracing::warn!(track = %track.name, error = ?e, "track error, marking finished"); + } + Poll::Pending => {} + } + } + } + + // Find minimum timestamp across pending frames + let mut min_idx = None; + let mut min_ts = None; + + for (i, track) in self.tracks.iter().enumerate() { + if let Some(frame) = &track.pending { + let ts: std::time::Duration = frame.timestamp.into(); + if min_ts.is_none() || ts < min_ts.unwrap() { + min_ts = Some(ts); + min_idx = Some(i); + } + } + } + + // Return the frame with the smallest timestamp + if let Some(idx) = min_idx { + let track = &mut self.tracks[idx]; + let frame = track.pending.take().unwrap(); + let name = track.name.clone(); + return Poll::Ready(Ok(Some(MuxedFrame { name, frame }))); + } + + // All finished + no pending → None + if self.tracks.iter().all(|t| t.finished) { + return Poll::Ready(Ok(None)); + } + + // Still waiting for data + Poll::Pending + } +} diff --git a/rs/moq-mux/src/consumer/ordered.rs b/rs/moq-mux/src/consumer/ordered.rs new file mode 100644 index 000000000..cb1f8a94d --- /dev/null +++ b/rs/moq-mux/src/consumer/ordered.rs @@ -0,0 +1,1192 @@ +use std::collections::VecDeque; +use std::task::{Poll, ready}; + +use buf_list::BufList; + +use super::container::Error; +use super::{ContainerFormat, OrderedFrame, Timestamp}; + +/// A consumer for media tracks with timestamp reordering. +/// +/// This wraps a `moq_lite::TrackConsumer` and adds functionality +/// like timestamp decoding, latency management, and frame buffering. +/// +/// Generic over `F: ContainerFormat` to support different container encodings. +/// +/// ## Latency Management +/// +/// The consumer can skip groups that are too far behind to maintain low latency. +/// Configure the maximum acceptable delay through the consumer's latency settings. +pub struct OrderedConsumer { + pub track: moq_lite::TrackConsumer, + + format: F, + + // The current group that we want to read from + current: u64, + + // Groups that we are monitoring, sorted by sequence ascending. + pending: VecDeque, + + // When true, we haven't returned a frame yet and need to select the first group. + // We wait until we have at least one frame before finalizing `current` + startup: bool, + + // The maximum buffer size before skipping a group. + max_latency: std::time::Duration, +} + +impl OrderedConsumer { + /// Create a new OrderedConsumer wrapping the given moq-lite consumer. + pub fn new(track: moq_lite::TrackConsumer, format: F, max_latency: std::time::Duration) -> Self { + Self { + track, + format, + current: 0, + pending: VecDeque::new(), + startup: true, + max_latency, + } + } + + /// Read the next frame from the track. + /// + /// This method handles timestamp decoding, group ordering, and latency management + /// automatically. It will skip groups that are too far behind to maintain the + /// configured latency target. + /// + /// Returns `None` when the track has ended. + pub async fn read(&mut self) -> Result, Error> { + conducer::wait(|waiter| self.poll_read(waiter)).await + } + + /// Poll-based implementation of the read loop. + /// + /// Uses a single waiter that gets registered on all relevant conducer channels, + /// avoiding the need for `tokio::select!` or `FuturesUnordered`. + pub fn poll_read(&mut self, waiter: &conducer::Waiter) -> Poll, Error>> { + // Grab any new groups from the track, recording whether the track is finished. + let finished = self.poll_read_finish(waiter)?.is_ready(); + + // On startup, we want to poll every pending group and advance self.current to the first with a frame. + if self.startup { + // NOTE: We loop in ascending order, so earlier groups will win the race. + for (i, group) in self.pending.iter_mut().enumerate() { + // We call poll_min_timestamp to try to buffer at least one frame per group. + // This returns Ready(Ok) if there is a buffered frame. + if !matches!(group.poll_min_timestamp(waiter, &self.format), Poll::Ready(Ok(_))) { + continue; + } + + // Start reading from this group and skip any previous groups. + self.current = group.info.sequence; + self.startup = false; + self.pending.drain(0..i); + break; + } + } + + loop { + // Return the next frame from the current group if possible. + // If the current group is finished or errored, advance to the next group. + while let Some(group) = self.pending.front_mut() + && group.info.sequence <= self.current + { + match group.poll_read(waiter, &self.format) { + Poll::Ready(Ok(Some(frame))) => return Poll::Ready(Ok(Some(frame))), + // Still blocked on this group, don't skip it yet. + Poll::Pending => break, + Poll::Ready(Err(e)) => { + tracing::warn!(error = ?e, "error reading current group, skipping"); + } + // No more frames, advance to next group. + Poll::Ready(Ok(None)) => {} + } + + self.pending.pop_front(); + self.current += 1 + } + + // Get the current group's min timestamp as the reference for latency comparison. + let oldest_timestamp = if let Some(current) = self.pending.front_mut() + && current.info.sequence <= self.current + { + match current.poll_min_timestamp(waiter, &self.format) { + Poll::Ready(Ok(ts)) => Some::(ts.into()), + _ => None, + } + } else { + None + }; + + // Find the first newer group with data (our skip target). + let mut min_idx = None; + for (i, group) in self.pending.iter_mut().enumerate() { + if group.info.sequence <= self.current { + continue; + } + + if let Poll::Ready(Ok(_)) = group.poll_min_timestamp(waiter, &self.format) { + min_idx = Some(i); + break; + } + } + + // Find the max timestamp across all newer groups. + let mut max_timestamp = std::time::Duration::ZERO; + for group in self.pending.iter_mut().rev() { + if group.info.sequence <= self.current { + break; + } + + if let Poll::Ready(Ok(ts)) = group.poll_max_timestamp(waiter, &self.format) { + max_timestamp = max_timestamp.max(ts.into()); + break; // We know older groups won't be newer than this. + } + } + + let should_skip = if min_idx.is_some() { + if let Some(oldest) = oldest_timestamp { + // Current group is blocking: skip if newer groups exceed latency threshold + max_timestamp.saturating_sub(oldest) >= self.max_latency + } else { + // Sequence gap: current group consumed but next sequence missing. + // Only skip if track is fully received (no more groups coming). + finished + } + } else { + false + }; + + if let Some(new_idx) = min_idx + && should_skip + { + self.pending.drain(0..new_idx); + let new_current = self.pending.front().map(|g| g.info.sequence).unwrap(); + + tracing::debug!(old = self.current, new = new_current, "skipping slow groups"); + + self.current = new_current; + continue; + } + + if finished && self.pending.is_empty() { + return Poll::Ready(Ok(None)); + } + + return Poll::Pending; + } + } + + // Reads any new groups from the track until we're completely finished. + // + // Returns Pending until all groups have been consumed. + fn poll_read_finish(&mut self, waiter: &conducer::Waiter) -> Poll> { + loop { + let Some(group) = ready!(self.track.poll_recv_group(waiter)?) else { + // Track is finished. + return Poll::Ready(Ok(())); + }; + + let reader = GroupBuffer::new(group); + if reader.group.info.sequence < self.current { + tracing::debug!( + old = ?reader.group.info.sequence, + current = ?self.current, + "skipping old group" + ); + continue; + } + + let idx = self + .pending + .partition_point(|g| g.group.info.sequence < reader.group.info.sequence); + self.pending.insert(idx, reader); + } + } + + /// Set the maximum latency tolerance for this consumer. + /// + /// Groups with timestamps older than `max_timestamp - max_latency` will be skipped. + pub fn set_max_latency(&mut self, max: std::time::Duration) { + self.max_latency = max; + } + + /// Wait until the track is closed. + pub async fn closed(&self) -> Result<(), Error> { + Ok(self.track.closed().await?) + } +} + +impl From> for moq_lite::TrackConsumer { + fn from(inner: OrderedConsumer) -> Self { + inner.track + } +} + +impl std::ops::Deref for OrderedConsumer { + type Target = moq_lite::TrackConsumer; + + fn deref(&self) -> &Self::Target { + &self.track + } +} + +/// Internal reader for a group of frames. +/// +/// Handles two-phase frame reading (get FrameConsumer, then read all data), +/// timestamp parsing, and min/max timestamp tracking for latency decisions. +struct GroupBuffer { + group: moq_lite::GroupConsumer, + + // The current frame index within the group. + index: usize, + + // Read frames that haven't been consumed yet. + buffered: VecDeque, + + // The minimum timestamp in the group. + min_timestamp: Option, + + // The maximum timestamp in the group. + max_timestamp: Option, +} + +impl GroupBuffer { + fn new(group: moq_lite::GroupConsumer) -> Self { + Self { + group, + index: 0, + buffered: VecDeque::new(), + max_timestamp: None, + min_timestamp: None, + } + } + + /// Poll for the next frame from this group. + fn poll_read( + &mut self, + waiter: &conducer::Waiter, + format: &F, + ) -> Poll, Error>> { + if let Some(frame) = self.buffered.pop_front() { + return Poll::Ready(Ok(Some(frame))); + } + + match ready!(self.buffer_one(waiter, format)?) { + true => Poll::Ready(Ok(Some(self.buffered.pop_front().unwrap()))), + false => Poll::Ready(Ok(None)), + } + } + + // Add one more frame to the buffer if possible. + // + // Returns false if the track is finished. + fn buffer_once(&mut self, waiter: &conducer::Waiter, format: &F) -> Poll> { + let Some(chunks) = ready!(self.group.poll_read_frame_chunks(waiter)?) else { + return Poll::Ready(Ok(false)); + }; + + let payload = BufList::from_iter(chunks); + let (timestamp, payload) = format.parse(payload).map_err(Into::into)?; + + self.min_timestamp = Some(match self.min_timestamp { + Some(existing) => existing.min(timestamp), + None => timestamp, + }); + + self.max_timestamp = Some(match self.max_timestamp { + Some(existing) => existing.max(timestamp), + None => timestamp, + }); + + let index = self.index; + self.index += 1; + + self.buffered.push_back(OrderedFrame { + timestamp, + payload, + group: self.group.info.sequence, + index, + }); + + Poll::Ready(Ok(true)) + } + + fn buffer_one(&mut self, waiter: &conducer::Waiter, format: &F) -> Poll> { + if self.buffered.is_empty() { + self.buffer_once(waiter, format) + } else { + Poll::Ready(Ok(true)) + } + } + + fn buffer_all(&mut self, waiter: &conducer::Waiter, format: &F) -> Poll> { + while ready!(self.buffer_once(waiter, format)?) {} + Poll::Ready(Ok(())) + } + + /// Poll for the maximum timestamp in this group. + fn poll_max_timestamp( + &mut self, + waiter: &conducer::Waiter, + format: &F, + ) -> Poll> { + // Keep reading more frames just to advance the max timestamp. + let _ = self.buffer_all(waiter, format)?; + + if let Some(max) = self.max_timestamp { + return Poll::Ready(Ok(max)); + } + + if let Poll::Ready(_frames) = self.group.poll_finished(waiter)? { + return Poll::Ready(Err(Error::Other("empty group".into()))); + } + + Poll::Pending + } + + fn poll_min_timestamp( + &mut self, + waiter: &conducer::Waiter, + format: &F, + ) -> Poll> { + let _ = self.buffer_one(waiter, format)?; + + if let Some(min) = self.min_timestamp { + return Poll::Ready(Ok(min)); + } + + if let Poll::Ready(_frames) = self.group.poll_finished(waiter)? { + return Poll::Ready(Err(Error::Other("empty group".into()))); + } + + Poll::Pending + } +} + +impl std::ops::Deref for GroupBuffer { + type Target = moq_lite::GroupConsumer; + + fn deref(&self) -> &Self::Target { + &self.group + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::consumer::Legacy; + use std::time::Duration; + + use bytes::Bytes; + use hang::container::Frame; + + fn ts(micros: u64) -> Timestamp { + Timestamp::from_micros(micros).unwrap() + } + + /// Write a finished group with explicit sequence and timestamps (Legacy format). + fn write_group(track: &mut moq_lite::TrackProducer, sequence: u64, timestamps: &[Timestamp]) { + let mut group = track.create_group(moq_lite::Group { sequence }).unwrap(); + for ×tamp in timestamps { + let frame = Frame { + timestamp, + payload: BufList::from_iter(vec![Bytes::from_static(&[0xDE, 0xAD])]), + }; + frame.encode(&mut group).unwrap(); + } + group.finish().unwrap(); + } + + /// Drain all available frames with a per-read timeout. + async fn read_all(consumer: &mut OrderedConsumer) -> Result, Error> { + let mut frames = Vec::new(); + loop { + match tokio::time::timeout(Duration::from_millis(200), consumer.read()).await { + Ok(Ok(Some(frame))) => frames.push(frame), + Ok(Ok(None)) => break, + Ok(Err(e)) => return Err(e), + Err(_) => panic!( + "read_all: OrderedConsumer::read timed out after 200ms ({} frames collected so far)", + frames.len() + ), + } + } + Ok(frames) + } + + // ---- Basic Reading ---- + + #[tokio::test] + async fn read_single_group() { + let mut track = moq_lite::Track::new("test").produce(); + let consumer_track = track.consume(); + let mut consumer = OrderedConsumer::new(consumer_track, Legacy, Duration::from_millis(500)); + + write_group(&mut track, 0, &[ts(0)]); + track.finish().unwrap(); + + let frames = read_all(&mut consumer).await.unwrap(); + assert_eq!(frames.len(), 1); + assert_eq!(frames[0].timestamp, ts(0)); + assert_eq!(frames[0].index, 0); + + // Next read returns None (track ended) + assert!(consumer.read().await.unwrap().is_none()); + } + + #[tokio::test] + async fn read_multiple_frames_single_group() { + let mut track = moq_lite::Track::new("test").produce(); + let consumer_track = track.consume(); + let mut consumer = OrderedConsumer::new(consumer_track, Legacy, Duration::from_millis(500)); + + write_group(&mut track, 0, &[ts(0), ts(33_000), ts(66_000)]); + track.finish().unwrap(); + + let frames = read_all(&mut consumer).await.unwrap(); + assert_eq!(frames.len(), 3); + assert_eq!(frames[0].timestamp, ts(0)); + assert_eq!(frames[1].timestamp, ts(33_000)); + assert_eq!(frames[2].timestamp, ts(66_000)); + + assert_eq!(frames[0].index, 0); + assert_eq!(frames[1].index, 1); + assert_eq!(frames[2].index, 2); + } + + #[tokio::test] + async fn read_multiple_groups_within_latency() { + let mut track = moq_lite::Track::new("test").produce(); + let consumer_track = track.consume(); + let mut consumer = OrderedConsumer::new(consumer_track, Legacy, Duration::from_millis(500)); + + // 5 groups, 20ms spacing. Total span = 80ms, well within 500ms latency. + for i in 0..5u64 { + write_group(&mut track, i, &[ts(i * 20_000)]); + } + track.finish().unwrap(); + + let frames = read_all(&mut consumer).await.unwrap(); + assert_eq!(frames.len(), 5); + } + + // ---- Latency Skipping ---- + + #[tokio::test] + async fn latency_skip_delivers_recent_groups() { + tokio::time::pause(); + let mut track = moq_lite::Track::new("test").produce(); + let consumer_track = track.consume(); + let mut consumer = OrderedConsumer::new(consumer_track, Legacy, Duration::from_millis(100)); + + // Group 0: 5 frames, NOT finished (blocks consumer) + let mut group0 = track.create_group(moq_lite::Group { sequence: 0 }).unwrap(); + for f in 0..5u64 { + Frame { + timestamp: ts(f * 2_000), + payload: BufList::from_iter(vec![Bytes::from_static(&[0xDE, 0xAD])]), + } + .encode(&mut group0) + .unwrap(); + } + + // Groups 1-19: finished, 15ms spacing, 5 frames each + for g in 1..20u64 { + let timestamps: Vec<_> = (0..5).map(|f| ts(g * 15_000 + f * 2_000)).collect(); + write_group(&mut track, g, ×tamps); + } + track.finish().unwrap(); + + // Finish group 0 after consumer has had time to accumulate pending groups + let finisher = tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(50)).await; + group0.finish().unwrap(); + }); + + let frames = read_all(&mut consumer).await.unwrap(); + // Group 0's 5 frames + some later groups (earlier ones skipped by latency) + assert!(frames.len() >= 25, "Expected >= 25 frames, got {}", frames.len()); + finisher.await.expect("finisher task panicked"); + } + + #[tokio::test] + async fn zero_latency_skips_aggressively() { + tokio::time::pause(); + let mut track = moq_lite::Track::new("test").produce(); + let consumer_track = track.consume(); + let mut consumer = OrderedConsumer::new(consumer_track, Legacy, Duration::ZERO); + + let mut group0 = track.create_group(moq_lite::Group { sequence: 0 }).unwrap(); + Frame { + timestamp: ts(400_000), + payload: BufList::from_iter(vec![Bytes::from_static(&[0xDE, 0xAD])]), + } + .encode(&mut group0) + .unwrap(); + + for g in 1..10u64 { + let timestamps: Vec<_> = (0..3).map(|f| ts(g * 50_000 + f * 5_000)).collect(); + write_group(&mut track, g, ×tamps); + } + track.finish().unwrap(); + + let finisher = tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(50)).await; + group0.finish().unwrap(); + }); + + let frames = read_all(&mut consumer).await.unwrap(); + assert_eq!(frames.len(), 28, "Expected group 0 frame + groups 1-9"); + assert!(!frames.is_empty(), "Expected at least some frames"); + finisher.await.expect("finisher task panicked"); + } + + #[tokio::test] + async fn latency_skip_correctness() { + tokio::time::pause(); + let mut track = moq_lite::Track::new("test").produce(); + let consumer_track = track.consume(); + let mut consumer = OrderedConsumer::new(consumer_track, Legacy, Duration::from_millis(100)); + + let mut group0 = track.create_group(moq_lite::Group { sequence: 0 }).unwrap(); + Frame { + timestamp: ts(0), + payload: BufList::from_iter(vec![Bytes::from_static(&[0xDE, 0xAD])]), + } + .encode(&mut group0) + .unwrap(); + + for g in 1..10u64 { + write_group(&mut track, g, &[ts(g * 30_000)]); + } + track.finish().unwrap(); + + let finisher = tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(50)).await; + group0.finish().unwrap(); + }); + + let frames = read_all(&mut consumer).await.unwrap(); + assert!(!frames.is_empty(), "Expected at least some frames"); + assert_eq!(frames.len(), 10, "Expected group 0 frame + groups 1-9"); + assert_eq!(frames[0].timestamp, ts(0)); + + for i in 1..10u64 { + assert_eq!(frames[i as usize].timestamp, ts(i * 30_000)); + } + finisher.await.expect("finisher task panicked"); + } + + // ---- Group Ordering ---- + + #[tokio::test] + async fn groups_delivered_in_sequence_order() { + tokio::time::pause(); + let mut track = moq_lite::Track::new("test").produce(); + let consumer_track = track.consume(); + let mut consumer = OrderedConsumer::new(consumer_track, Legacy, Duration::from_millis(500)); + + let mut group0 = track.create_group(moq_lite::Group { sequence: 0 }).unwrap(); + Frame { + timestamp: ts(0), + payload: BufList::from_iter(vec![Bytes::from_static(&[0xDE, 0xAD])]), + } + .encode(&mut group0) + .unwrap(); + + write_group(&mut track, 2, &[ts(60_000)]); + write_group(&mut track, 1, &[ts(30_000)]); + track.finish().unwrap(); + + let finisher = tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(10)).await; + group0.finish().unwrap(); + }); + + let frames = read_all(&mut consumer).await.unwrap(); + assert_eq!(frames.len(), 3); + assert_eq!(frames[0].timestamp, ts(0)); + assert_eq!(frames[1].timestamp, ts(30_000)); + assert_eq!(frames[2].timestamp, ts(60_000)); + finisher.await.expect("finisher task panicked"); + } + + #[tokio::test] + async fn adjacent_group_flushed_immediately() { + let mut track = moq_lite::Track::new("test").produce(); + let consumer_track = track.consume(); + let mut consumer = OrderedConsumer::new(consumer_track, Legacy, Duration::from_millis(500)); + + write_group(&mut track, 0, &[ts(0)]); + write_group(&mut track, 1, &[ts(30_000)]); + track.finish().unwrap(); + + let frames = read_all(&mut consumer).await.unwrap(); + assert_eq!(frames.len(), 2); + assert_eq!(frames[0].timestamp, ts(0)); + assert_eq!(frames[1].timestamp, ts(30_000)); + } + + // ---- B-frames ---- + + #[tokio::test] + async fn bframes_within_group() { + let mut track = moq_lite::Track::new("test").produce(); + let consumer_track = track.consume(); + let mut consumer = OrderedConsumer::new(consumer_track, Legacy, Duration::from_millis(500)); + + write_group(&mut track, 0, &[ts(0), ts(66_000), ts(33_000)]); + track.finish().unwrap(); + + let frames = read_all(&mut consumer).await.unwrap(); + assert_eq!(frames.len(), 3); + assert_eq!(frames[0].timestamp, ts(0)); + assert_eq!(frames[1].timestamp, ts(66_000)); + assert_eq!(frames[2].timestamp, ts(33_000)); + } + + // ---- Track Lifecycle ---- + + #[tokio::test] + async fn empty_track_returns_none() { + tokio::time::pause(); + let mut track = moq_lite::Track::new("test").produce(); + let consumer_track = track.consume(); + let mut consumer = OrderedConsumer::new(consumer_track, Legacy, Duration::from_millis(500)); + + track.finish().unwrap(); + + let result = tokio::time::timeout(Duration::from_millis(200), consumer.read()).await; + match result { + Ok(Ok(None)) => {} // expected: track ended + Ok(Ok(Some(_))) => panic!("expected None for empty track, got Some"), + Ok(Err(e)) => panic!("expected None for empty track, got error: {e}"), + Err(_) => panic!("should not hang on empty track"), + } + } + + #[tokio::test] + async fn track_closed_with_error() { + tokio::time::pause(); + let mut track = moq_lite::Track::new("test").produce(); + let consumer_track = track.consume(); + let mut consumer = OrderedConsumer::new(consumer_track, Legacy, Duration::from_millis(500)); + + write_group(&mut track, 0, &[ts(0)]); + track.abort(moq_lite::Error::Cancel).unwrap(); + + let result = tokio::time::timeout(Duration::from_millis(500), async { + let mut frames = Vec::new(); + while let Ok(Some(frame)) = consumer.read().await { + frames.push(frame); + } + frames + }) + .await; + + assert!(result.is_ok(), "Consumer should not hang after track error"); + } + + #[tokio::test] + async fn closed_resolves_when_track_ends() { + tokio::time::pause(); + let mut track = moq_lite::Track::new("test").produce(); + let consumer_track = track.consume(); + let consumer = OrderedConsumer::new(consumer_track, Legacy, Duration::from_millis(500)); + + assert!( + tokio::time::timeout(Duration::from_millis(50), consumer.closed()) + .await + .is_err() + ); + + track.finish().unwrap(); + drop(track); + + tokio::time::timeout(Duration::from_millis(200), consumer.closed()) + .await + .expect("timeout expired waiting for closed()") + .expect("consumer.closed() returned an error"); + } + + // ---- Gap Recovery ---- + + #[tokio::test] + async fn gap_in_group_sequence_recovery() { + let mut track = moq_lite::Track::new("test").produce(); + let consumer_track = track.consume(); + let mut consumer = OrderedConsumer::new(consumer_track, Legacy, Duration::from_millis(100)); + + write_group(&mut track, 0, &[ts(0), ts(20_000)]); + write_group(&mut track, 1, &[ts(40_000), ts(60_000)]); + write_group(&mut track, 3, &[ts(120_000), ts(140_000)]); + write_group(&mut track, 4, &[ts(160_000), ts(180_000)]); + write_group(&mut track, 5, &[ts(200_000), ts(220_000)]); + write_group(&mut track, 6, &[ts(240_000), ts(260_000)]); + track.finish().unwrap(); + + let frames = read_all(&mut consumer).await.unwrap(); + assert!(frames.len() >= 4, "Expected >= 4 frames, got {}", frames.len()); + } + + #[tokio::test] + async fn gap_at_start_of_sequence() { + let mut track = moq_lite::Track::new("test").produce(); + let consumer_track = track.consume(); + let mut consumer = OrderedConsumer::new(consumer_track, Legacy, Duration::from_millis(80)); + + write_group(&mut track, 5, &[ts(0), ts(20_000)]); + write_group(&mut track, 7, &[ts(80_000), ts(100_000)]); + write_group(&mut track, 8, &[ts(120_000), ts(140_000)]); + write_group(&mut track, 9, &[ts(160_000), ts(180_000)]); + track.finish().unwrap(); + + let frames = read_all(&mut consumer).await.unwrap(); + assert!(frames.len() >= 4, "Expected >= 4 frames, got {}", frames.len()); + } + + // ---- Frame Decoding ---- + + #[tokio::test] + async fn frame_timestamp_and_index_decoding() { + let mut track = moq_lite::Track::new("test").produce(); + let consumer_track = track.consume(); + let mut consumer = OrderedConsumer::new(consumer_track, Legacy, Duration::from_millis(500)); + + write_group(&mut track, 0, &[ts(0), ts(33_333), ts(66_666)]); + track.finish().unwrap(); + + let frames = read_all(&mut consumer).await.unwrap(); + assert_eq!(frames.len(), 3); + + assert_eq!(frames[0].timestamp, ts(0)); + assert_eq!(frames[0].index, 0); + + assert_eq!(frames[1].timestamp, ts(33_333)); + assert_eq!(frames[1].index, 1); + + assert_eq!(frames[2].timestamp, ts(66_666)); + assert_eq!(frames[2].index, 2); + } + + #[tokio::test] + async fn frame_payload_preserved() { + let mut track = moq_lite::Track::new("test").produce(); + let consumer_track = track.consume(); + let mut consumer = OrderedConsumer::new(consumer_track, Legacy, Duration::from_millis(500)); + + let payload_bytes = vec![0x01, 0x02, 0x03, 0x04, 0x05]; + let mut group = track.create_group(moq_lite::Group { sequence: 0 }).unwrap(); + Frame { + timestamp: ts(0), + payload: BufList::from_iter(vec![Bytes::from(payload_bytes.clone())]), + } + .encode(&mut group) + .unwrap(); + group.finish().unwrap(); + track.finish().unwrap(); + + let frames = read_all(&mut consumer).await.unwrap(); + assert_eq!(frames.len(), 1); + + use bytes::Buf; + let mut received = Vec::new(); + let mut payload = frames[0].payload.clone(); + while payload.has_remaining() { + received.push(payload.get_u8()); + } + assert_eq!(received, payload_bytes); + } + + // ---- Regression ---- + + #[tokio::test] + async fn no_infinite_loop_with_buffered_frames() { + tokio::time::pause(); + let mut track = moq_lite::Track::new("test").produce(); + let consumer_track = track.consume(); + let mut consumer = OrderedConsumer::new(consumer_track, Legacy, Duration::from_secs(10)); + + let mut group0 = track.create_group(moq_lite::Group { sequence: 0 }).unwrap(); + Frame { + timestamp: ts(0), + payload: BufList::from_iter(vec![Bytes::from_static(&[0xDE, 0xAD])]), + } + .encode(&mut group0) + .unwrap(); + + write_group(&mut track, 1, &[ts(100_000)]); + + let finisher = tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(20)).await; + // Write group 2: recv_group fires, drops current buffer_until for group 1 + write_group(&mut track, 2, &[ts(200_000)]); + tokio::time::sleep(Duration::from_millis(20)).await; + group0.finish().unwrap(); + track.finish().unwrap(); + }); + + let frames = tokio::time::timeout(Duration::from_secs(2), async { + let mut frames = Vec::new(); + while let Some(frame) = consumer.read().await.unwrap() { + frames.push(frame); + } + frames + }) + .await + .expect("consumer hung — possible infinite loop regression"); + + assert_eq!(frames.len(), 3); + finisher.await.expect("finisher task panicked"); + } + + // ---- Edge Cases ---- + + #[tokio::test] + async fn large_timestamps() { + let mut track = moq_lite::Track::new("test").produce(); + let consumer_track = track.consume(); + let mut consumer = OrderedConsumer::new(consumer_track, Legacy, Duration::from_secs(3700)); + + let one_hour = 3_600_000_000u64; + write_group(&mut track, 0, &[ts(one_hour)]); + track.finish().unwrap(); + + let frames = read_all(&mut consumer).await.unwrap(); + assert_eq!(frames.len(), 1); + assert_eq!(frames[0].timestamp, ts(one_hour)); + assert_eq!(frames[0].timestamp.as_micros(), one_hour as u128); + } + + #[tokio::test] + async fn set_max_latency_changes_behavior() { + let mut track = moq_lite::Track::new("test").produce(); + let consumer_track = track.consume(); + let mut consumer = OrderedConsumer::new(consumer_track, Legacy, Duration::from_secs(10)); + + write_group(&mut track, 0, &[ts(0)]); + track.finish().unwrap(); + + let frame = consumer.read().await.unwrap().unwrap(); + assert_eq!(frame.timestamp, ts(0)); + + consumer.set_max_latency(Duration::from_millis(100)); + + assert!(consumer.read().await.unwrap().is_none()); + } + + #[tokio::test] + async fn max_timestamp_tracks_through_bframes() { + tokio::time::pause(); + let mut track = moq_lite::Track::new("test").produce(); + let consumer_track = track.consume(); + // max_latency must exceed (group1_max - group0_min) = 100ms - 0ms = 100ms + // to avoid the latency skip and test B-frame timestamp tracking. + let mut consumer = OrderedConsumer::new(consumer_track, Legacy, Duration::from_millis(110)); + + let mut group0 = track.create_group(moq_lite::Group { sequence: 0 }).unwrap(); + for ×tamp in &[ts(0), ts(66_000), ts(33_000)] { + Frame { + timestamp, + payload: BufList::from_iter(vec![Bytes::from_static(&[0xDE, 0xAD])]), + } + .encode(&mut group0) + .unwrap(); + } + + write_group(&mut track, 1, &[ts(100_000)]); + track.finish().unwrap(); + + let finisher = tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(50)).await; + group0.finish().unwrap(); + }); + + let frames = tokio::time::timeout(Duration::from_secs(2), async { + let mut frames = Vec::new(); + while let Some(frame) = consumer.read().await.unwrap() { + frames.push(frame); + } + frames + }) + .await + .expect("consumer hung — max_timestamp regression"); + + assert_eq!(frames.len(), 4, "Expected all 4 frames, got {}", frames.len()); + assert_eq!(frames[0].timestamp, ts(0)); + assert_eq!(frames[1].timestamp, ts(66_000)); + assert_eq!(frames[2].timestamp, ts(33_000)); + assert_eq!(frames[3].timestamp, ts(100_000)); + finisher.await.expect("finisher task panicked"); + } + + // ---- Startup Behavior ---- + + #[tokio::test] + async fn startup_selects_earliest_group() { + tokio::time::pause(); + let mut track = moq_lite::Track::new("test").produce(); + let consumer_track = track.consume(); + let mut consumer = OrderedConsumer::new(consumer_track, Legacy, Duration::from_millis(100)); + + write_group(&mut track, 3, &[ts(0)]); + write_group(&mut track, 5, &[ts(150_000)]); + + let mut group7 = track.create_group(moq_lite::Group { sequence: 7 }).unwrap(); + Frame { + timestamp: ts(300_000), + payload: BufList::from_iter(vec![Bytes::from_static(&[0xDE, 0xAD])]), + } + .encode(&mut group7) + .unwrap(); + + let finisher = tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(50)).await; + Frame { + timestamp: ts(400_000), + payload: BufList::from_iter(vec![Bytes::from_static(&[0xBE, 0xEF])]), + } + .encode(&mut group7) + .unwrap(); + group7.finish().unwrap(); + track.finish().unwrap(); + }); + + let frames = tokio::time::timeout(Duration::from_secs(2), async { + let mut frames = Vec::new(); + while let Some(frame) = consumer.read().await.unwrap() { + frames.push(frame); + } + frames + }) + .await + .expect("should not hang"); + + assert_eq!(frames[0].group, 3); + assert_eq!(frames[1].group, 5); + assert!(frames.iter().skip(2).all(|f| f.group == 7)); + finisher.await.unwrap(); + } + + #[tokio::test] + async fn startup_skips_groups_without_data() { + tokio::time::pause(); + let mut track = moq_lite::Track::new("test").produce(); + let consumer_track = track.consume(); + let mut consumer = OrderedConsumer::new(consumer_track, Legacy, Duration::from_millis(500)); + + let _group5 = track.create_group(moq_lite::Group { sequence: 5 }).unwrap(); + write_group(&mut track, 7, &[ts(210_000)]); + track.finish().unwrap(); + + let frames = tokio::time::timeout(Duration::from_millis(500), async { + let mut frames = Vec::new(); + while let Some(frame) = consumer.read().await.unwrap() { + frames.push(frame); + } + frames + }) + .await + .expect("should not hang"); + + assert!(!frames.is_empty()); + assert_eq!(frames[0].group, 7); + } + + #[tokio::test] + async fn startup_single_group_mid_stream() { + let mut track = moq_lite::Track::new("test").produce(); + let consumer_track = track.consume(); + let mut consumer = OrderedConsumer::new(consumer_track, Legacy, Duration::from_millis(500)); + + write_group(&mut track, 100, &[ts(3_000_000)]); + track.finish().unwrap(); + + let frames = read_all(&mut consumer).await.unwrap(); + assert_eq!(frames.len(), 1); + assert_eq!(frames[0].group, 100); + } + + #[tokio::test] + async fn multiple_sequential_latency_skips() { + tokio::time::pause(); + let mut track = moq_lite::Track::new("test").produce(); + let consumer_track = track.consume(); + let mut consumer = OrderedConsumer::new(consumer_track, Legacy, Duration::from_millis(50)); + + let mut group0 = track.create_group(moq_lite::Group { sequence: 0 }).unwrap(); + Frame { + timestamp: ts(0), + payload: BufList::from_iter(vec![Bytes::from_static(&[0xAA])]), + } + .encode(&mut group0) + .unwrap(); + + write_group(&mut track, 1, &[ts(100_000)]); + write_group(&mut track, 2, &[ts(200_000)]); + write_group(&mut track, 3, &[ts(300_000)]); + track.finish().unwrap(); + + let finisher = tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(20)).await; + group0.finish().unwrap(); + }); + + let frames = read_all(&mut consumer).await.unwrap(); + assert!(!frames.is_empty()); + finisher.await.unwrap(); + } + + #[tokio::test] + async fn latency_skip_boundary_exact() { + tokio::time::pause(); + let mut track = moq_lite::Track::new("test").produce(); + let consumer_track = track.consume(); + let mut consumer = OrderedConsumer::new(consumer_track, Legacy, Duration::from_millis(100)); + + let mut group0 = track.create_group(moq_lite::Group { sequence: 0 }).unwrap(); + Frame { + timestamp: ts(0), + payload: BufList::from_iter(vec![Bytes::from_static(&[0xAA])]), + } + .encode(&mut group0) + .unwrap(); + + write_group(&mut track, 1, &[ts(100_000)]); + track.finish().unwrap(); + + let finisher = tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(20)).await; + group0.finish().unwrap(); + }); + + let frames = read_all(&mut consumer).await.unwrap(); + assert!(!frames.is_empty()); + finisher.await.unwrap(); + } + + /// Regression: a single stalled group with one newer group should trigger + /// a latency skip when the timestamp difference exceeds max_latency. + /// Previously, the span was computed across newer groups only (zero for one + /// group), so the skip never fired. + #[tokio::test] + async fn single_newer_group_triggers_skip() { + tokio::time::pause(); + let mut track = moq_lite::Track::new("test").produce(); + let consumer_track = track.consume(); + let mut consumer = OrderedConsumer::new(consumer_track, Legacy, Duration::from_millis(100)); + + // Group 0: stalled at ts=0, NOT finished + let mut group0 = track.create_group(moq_lite::Group { sequence: 0 }).unwrap(); + Frame { + timestamp: ts(0), + payload: BufList::from_iter(vec![Bytes::from_static(&[0xDE, 0xAD])]), + } + .encode(&mut group0) + .unwrap(); + + // Group 1: finished, 200ms ahead (well beyond 100ms max_latency) + write_group(&mut track, 1, &[ts(200_000)]); + track.finish().unwrap(); + + let finisher = tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(50)).await; + group0.finish().unwrap(); + }); + + let frames = read_all(&mut consumer).await.unwrap(); + assert_eq!(frames.len(), 2, "Expected group 0 frame + group 1 frame"); + assert_eq!(frames[0].group, 0); + assert_eq!(frames[1].group, 1); + finisher.await.unwrap(); + } + + /// Regression: when the current group is fully consumed and the next sequence + /// is missing (gap), the consumer should skip to the next available group + /// once the track is fully received, rather than hanging forever. + #[tokio::test] + async fn single_missing_sequence_near_eof_skips() { + tokio::time::pause(); + let mut track = moq_lite::Track::new("test").produce(); + let consumer_track = track.consume(); + let mut consumer = OrderedConsumer::new(consumer_track, Legacy, Duration::from_millis(100)); + + // Group 0: finished normally + write_group(&mut track, 0, &[ts(0), ts(20_000)]); + // Group 2: finished (group 1 is missing — sequence gap) + write_group(&mut track, 2, &[ts(200_000)]); + track.finish().unwrap(); + + let frames = read_all(&mut consumer).await.unwrap(); + assert_eq!(frames.len(), 3, "Expected group 0 (2 frames) + group 2 (1 frame)"); + assert_eq!(frames[0].group, 0); + assert_eq!(frames[1].group, 0); + assert_eq!(frames[2].group, 2); + } + + #[tokio::test] + async fn group_error_skips_to_next() { + let mut track = moq_lite::Track::new("test").produce(); + let consumer_track = track.consume(); + let mut consumer = OrderedConsumer::new(consumer_track, Legacy, Duration::from_millis(500)); + + let mut group0 = track.create_group(moq_lite::Group { sequence: 0 }).unwrap(); + group0.abort(moq_lite::Error::Cancel).unwrap(); + + write_group(&mut track, 1, &[ts(30_000)]); + track.finish().unwrap(); + + let frames = read_all(&mut consumer).await.unwrap(); + assert_eq!(frames.len(), 1); + assert_eq!(frames[0].group, 1); + } + + #[tokio::test] + async fn track_finishes_while_reading() { + tokio::time::pause(); + let mut track = moq_lite::Track::new("test").produce(); + let consumer_track = track.consume(); + let mut consumer = OrderedConsumer::new(consumer_track, Legacy, Duration::from_millis(500)); + + write_group(&mut track, 0, &[ts(0)]); + + let finisher = tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(20)).await; + write_group(&mut track, 1, &[ts(30_000)]); + tokio::time::sleep(Duration::from_millis(20)).await; + track.finish().unwrap(); + }); + + let frames = tokio::time::timeout(Duration::from_secs(2), async { + let mut frames = Vec::new(); + while let Some(frame) = consumer.read().await.unwrap() { + frames.push(frame); + } + frames + }) + .await + .expect("consumer should not hang"); + + assert_eq!(frames.len(), 2); + finisher.await.unwrap(); + } + + #[tokio::test] + async fn empty_group_advances() { + let mut track = moq_lite::Track::new("test").produce(); + let consumer_track = track.consume(); + let mut consumer = OrderedConsumer::new(consumer_track, Legacy, Duration::from_millis(500)); + + let mut group0 = track.create_group(moq_lite::Group { sequence: 0 }).unwrap(); + group0.finish().unwrap(); + + write_group(&mut track, 1, &[ts(30_000)]); + track.finish().unwrap(); + + let frames = read_all(&mut consumer).await.unwrap(); + assert_eq!(frames.len(), 1); + assert_eq!(frames[0].group, 1); + } +} diff --git a/rs/moq-mux/src/convert/fmp4.rs b/rs/moq-mux/src/convert/fmp4.rs new file mode 100644 index 000000000..b83c62cef --- /dev/null +++ b/rs/moq-mux/src/convert/fmp4.rs @@ -0,0 +1,511 @@ +use anyhow::Context; +use base64::Engine; +use bytes::Bytes; +use hang::catalog::{AudioCodec, AudioConfig, Container, VideoCodec, VideoConfig}; +use mp4_atom::{Atom, Encode}; + +/// Converts a broadcast from any format to CMAF format. +/// +/// If tracks are already CMAF, they are passed through unchanged. +/// If tracks are hang/Legacy, each frame is individually wrapped in moof+mdat. +pub struct Fmp4 { + input: moq_lite::BroadcastConsumer, + output: moq_lite::BroadcastProducer, +} + +impl Fmp4 { + pub fn new(input: moq_lite::BroadcastConsumer, output: moq_lite::BroadcastProducer) -> Self { + Self { input, output } + } + + /// Run the converter. + /// + /// Reads the hang catalog from the input broadcast. If tracks are already CMAF, + /// passes them through unchanged (no-op). If tracks are hang/Legacy, converts + /// each frame to moof+mdat. + pub async fn run(self) -> anyhow::Result<()> { + let mut broadcast = self.output; + let catalog_producer = crate::CatalogProducer::new(&mut broadcast)?; + + let catalog_track = self.input.subscribe_track(&hang::Catalog::default_track())?; + let mut catalog_consumer = hang::CatalogConsumer::new(catalog_track); + let catalog = catalog_consumer.next().await?.context("empty catalog")?; + + let mut output_catalog = catalog_producer.clone(); + let mut guard = output_catalog.lock(); + let mut tasks = tokio::task::JoinSet::new(); + + for (name, config) in &catalog.video.renditions { + let input_track = self.input.subscribe_track(&moq_lite::Track { + name: name.clone(), + priority: 1, + })?; + + match &config.container { + Container::Cmaf { .. } => { + guard.video.renditions.insert(name.clone(), config.clone()); + let output_track = broadcast.create_track(moq_lite::Track { + name: name.clone(), + priority: 1, + })?; + let track_name = name.clone(); + tasks.spawn(async move { + if let Err(e) = passthrough_track(input_track, output_track).await { + tracing::error!(%e, track = %track_name, "passthrough_track failed"); + } + }); + } + Container::Legacy => { + let init_data = build_video_init(config)?; + let timescale = guess_video_timescale(config); + + let mut cmaf_config = config.clone(); + cmaf_config.container = Container::Cmaf { + init_data: base64::engine::general_purpose::STANDARD.encode(&init_data), + }; + guard.video.renditions.insert(name.clone(), cmaf_config); + + let output_track = broadcast.create_track(moq_lite::Track { + name: name.clone(), + priority: 1, + })?; + + let track_name = name.clone(); + tasks.spawn(async move { + if let Err(e) = convert_legacy_to_cmaf(input_track, output_track, timescale, true).await { + tracing::error!(%e, track = %track_name, "convert_legacy_to_cmaf failed"); + } + }); + } + } + } + + for (name, config) in &catalog.audio.renditions { + let input_track = self.input.subscribe_track(&moq_lite::Track { + name: name.clone(), + priority: 2, + })?; + + match &config.container { + Container::Cmaf { .. } => { + guard.audio.renditions.insert(name.clone(), config.clone()); + let output_track = broadcast.create_track(moq_lite::Track { + name: name.clone(), + priority: 2, + })?; + let track_name = name.clone(); + tasks.spawn(async move { + if let Err(e) = passthrough_track(input_track, output_track).await { + tracing::error!(%e, track = %track_name, "passthrough_track failed"); + } + }); + } + Container::Legacy => { + let init_data = build_audio_init(config)?; + + let mut cmaf_config = config.clone(); + cmaf_config.container = Container::Cmaf { + init_data: base64::engine::general_purpose::STANDARD.encode(&init_data), + }; + guard.audio.renditions.insert(name.clone(), cmaf_config); + + let output_track = broadcast.create_track(moq_lite::Track { + name: name.clone(), + priority: 2, + })?; + + let timescale = config.sample_rate as u64; + let track_name = name.clone(); + tasks.spawn(async move { + if let Err(e) = convert_legacy_to_cmaf(input_track, output_track, timescale, false).await { + tracing::error!(%e, track = %track_name, "convert_legacy_to_cmaf failed"); + } + }); + } + } + } + + drop(guard); + + // Keep broadcast and catalog alive until all track tasks complete. + while tasks.join_next().await.is_some() {} + + Ok(()) + } +} + +async fn passthrough_track( + mut input: moq_lite::TrackConsumer, + mut output: moq_lite::TrackProducer, +) -> anyhow::Result<()> { + while let Some(group) = input.recv_group().await? { + let mut out_group = output.append_group()?; + let mut reader = group; + while let Some(data) = reader.read_frame().await? { + out_group.write_frame(data)?; + } + out_group.finish()?; + } + output.finish()?; + Ok(()) +} + +async fn convert_legacy_to_cmaf( + input: moq_lite::TrackConsumer, + mut output: moq_lite::TrackProducer, + timescale: u64, + is_video: bool, +) -> anyhow::Result<()> { + let mut consumer = crate::consumer::OrderedConsumer::new(input, crate::consumer::Legacy, std::time::Duration::MAX); + let mut seq: u32 = 1; + let mut current_group: Option = None; + + while let Some(frame) = consumer.read().await? { + let keyframe = frame.is_keyframe(); + + if is_video && keyframe { + if let Some(mut prev) = current_group.take() { + prev.finish()?; + } + current_group = Some(output.append_group()?); + } else if current_group.is_none() { + current_group = Some(output.append_group()?); + } + + let group = current_group.as_mut().unwrap(); + + let payload: Vec = frame.payload.into_iter().flat_map(|chunk| chunk.into_iter()).collect(); + let dts = frame.timestamp.as_micros() as u64 * timescale / 1_000_000; + let moof_mdat = build_moof_mdat(seq, 1, dts, &payload, keyframe)?; + seq += 1; + + group.write_frame(moof_mdat)?; + } + + if let Some(mut g) = current_group.take() { + g.finish()?; + } + output.finish()?; + Ok(()) +} + +pub(crate) fn build_moof_mdat(seq: u32, track_id: u32, dts: u64, data: &[u8], keyframe: bool) -> anyhow::Result { + let flags = if keyframe { 0x0200_0000 } else { 0x0001_0000 }; + + // First pass to get moof size (use Some(0) so trun includes the data_offset field) + let moof = build_moof(seq, track_id, dts, data.len() as u32, flags, Some(0)); + let mut buf = Vec::new(); + moof.encode(&mut buf)?; + let moof_size = buf.len(); + + // Second pass with data_offset + let data_offset = (moof_size + 8) as i32; // 8 = mdat header + let moof = build_moof(seq, track_id, dts, data.len() as u32, flags, Some(data_offset)); + buf.clear(); + moof.encode(&mut buf)?; + + let mdat = mp4_atom::Mdat { data: data.to_vec() }; + mdat.encode(&mut buf)?; + + Ok(Bytes::from(buf)) +} + +fn build_moof(seq: u32, track_id: u32, dts: u64, size: u32, flags: u32, data_offset: Option) -> mp4_atom::Moof { + mp4_atom::Moof { + mfhd: mp4_atom::Mfhd { sequence_number: seq }, + traf: vec![mp4_atom::Traf { + tfhd: mp4_atom::Tfhd { + track_id, + ..Default::default() + }, + tfdt: Some(mp4_atom::Tfdt { + base_media_decode_time: dts, + }), + trun: vec![mp4_atom::Trun { + data_offset, + entries: vec![mp4_atom::TrunEntry { + size: Some(size), + flags: Some(flags), + ..Default::default() + }], + }], + ..Default::default() + }], + } +} + +pub(crate) fn build_video_init(config: &VideoConfig) -> anyhow::Result> { + let ftyp = mp4_atom::Ftyp { + major_brand: b"isom".into(), + minor_version: 0x200, + compatible_brands: vec![b"isom".into(), b"iso6".into(), b"mp41".into()], + }; + + let codec = build_video_codec(config)?; + let timescale = guess_video_timescale(config) as u32; + + let moov = mp4_atom::Moov { + mvhd: mp4_atom::Mvhd { + timescale, + ..Default::default() + }, + trak: vec![mp4_atom::Trak { + tkhd: mp4_atom::Tkhd { + track_id: 1, + width: mp4_atom::FixedPoint::new(config.coded_width.unwrap_or(0) as u16, 0), + height: mp4_atom::FixedPoint::new(config.coded_height.unwrap_or(0) as u16, 0), + ..Default::default() + }, + mdia: mp4_atom::Mdia { + mdhd: mp4_atom::Mdhd { + timescale, + ..Default::default() + }, + hdlr: mp4_atom::Hdlr { + handler: b"vide".into(), + name: "VideoHandler".into(), + }, + minf: mp4_atom::Minf { + vmhd: Some(mp4_atom::Vmhd::default()), + dinf: mp4_atom::Dinf { + dref: mp4_atom::Dref { urls: vec![] }, + }, + stbl: mp4_atom::Stbl { + stsd: mp4_atom::Stsd { codecs: vec![codec] }, + ..Default::default() + }, + ..Default::default() + }, + }, + ..Default::default() + }], + mvex: Some(mp4_atom::Mvex { + trex: vec![mp4_atom::Trex { + track_id: 1, + default_sample_description_index: 1, + ..Default::default() + }], + ..Default::default() + }), + ..Default::default() + }; + + let mut buf = Vec::new(); + ftyp.encode(&mut buf)?; + moov.encode(&mut buf)?; + Ok(buf) +} + +pub(crate) fn build_audio_init(config: &AudioConfig) -> anyhow::Result> { + let ftyp = mp4_atom::Ftyp { + major_brand: b"isom".into(), + minor_version: 0x200, + compatible_brands: vec![b"isom".into(), b"iso6".into(), b"mp41".into()], + }; + + let codec = build_audio_codec(config)?; + let timescale = config.sample_rate; + + let moov = mp4_atom::Moov { + mvhd: mp4_atom::Mvhd { + timescale, + ..Default::default() + }, + trak: vec![mp4_atom::Trak { + tkhd: mp4_atom::Tkhd { + track_id: 1, + ..Default::default() + }, + mdia: mp4_atom::Mdia { + mdhd: mp4_atom::Mdhd { + timescale, + ..Default::default() + }, + hdlr: mp4_atom::Hdlr { + handler: b"soun".into(), + name: "SoundHandler".into(), + }, + minf: mp4_atom::Minf { + smhd: Some(mp4_atom::Smhd::default()), + dinf: mp4_atom::Dinf { + dref: mp4_atom::Dref { urls: vec![] }, + }, + stbl: mp4_atom::Stbl { + stsd: mp4_atom::Stsd { codecs: vec![codec] }, + ..Default::default() + }, + ..Default::default() + }, + }, + ..Default::default() + }], + mvex: Some(mp4_atom::Mvex { + trex: vec![mp4_atom::Trex { + track_id: 1, + default_sample_description_index: 1, + ..Default::default() + }], + ..Default::default() + }), + ..Default::default() + }; + + let mut buf = Vec::new(); + ftyp.encode(&mut buf)?; + moov.encode(&mut buf)?; + Ok(buf) +} + +fn build_video_codec(config: &VideoConfig) -> anyhow::Result { + let visual = mp4_atom::Visual { + width: config.coded_width.unwrap_or(0) as u16, + height: config.coded_height.unwrap_or(0) as u16, + ..Default::default() + }; + + match &config.codec { + VideoCodec::H264(_) => { + let mut data = config + .description + .as_ref() + .context("H264 requires description")? + .clone(); + let avcc = mp4_atom::Avcc::decode_body(&mut data)?; + Ok(mp4_atom::Codec::Avc1(mp4_atom::Avc1 { + visual, + avcc, + ..Default::default() + })) + } + VideoCodec::H265(h265) => { + let mut data = config + .description + .as_ref() + .context("H265 requires description")? + .clone(); + let hvcc = mp4_atom::Hvcc::decode_body(&mut data)?; + if h265.in_band { + Ok(mp4_atom::Codec::Hev1(mp4_atom::Hev1 { + visual, + hvcc, + ..Default::default() + })) + } else { + Ok(mp4_atom::Codec::Hvc1(mp4_atom::Hvc1 { + visual, + hvcc, + ..Default::default() + })) + } + } + VideoCodec::VP9(vp9) => Ok(mp4_atom::Codec::Vp09(mp4_atom::Vp09 { + visual, + vpcc: mp4_atom::VpcC { + profile: vp9.profile, + level: vp9.level, + bit_depth: vp9.bit_depth, + chroma_subsampling: vp9.chroma_subsampling, + video_full_range_flag: vp9.full_range, + color_primaries: vp9.color_primaries, + transfer_characteristics: vp9.transfer_characteristics, + matrix_coefficients: vp9.matrix_coefficients, + codec_initialization_data: vec![], + }, + ..Default::default() + })), + VideoCodec::AV1(av1) => Ok(mp4_atom::Codec::Av01(mp4_atom::Av01 { + visual, + av1c: mp4_atom::Av1c { + seq_profile: av1.profile, + seq_level_idx_0: av1.level, + seq_tier_0: av1.tier == 'H', + high_bitdepth: av1.bitdepth >= 10, + twelve_bit: av1.bitdepth >= 12, + monochrome: av1.mono_chrome, + chroma_subsampling_x: av1.chroma_subsampling_x, + chroma_subsampling_y: av1.chroma_subsampling_y, + chroma_sample_position: av1.chroma_sample_position, + ..Default::default() + }, + ..Default::default() + })), + VideoCodec::VP8 => Ok(mp4_atom::Codec::Vp08(mp4_atom::Vp08 { + visual, + ..Default::default() + })), + _ => anyhow::bail!("unsupported video codec for CMAF conversion"), + } +} + +fn build_audio_codec(config: &AudioConfig) -> anyhow::Result { + let audio = mp4_atom::Audio { + data_reference_index: 1, + channel_count: config.channel_count as u16, + sample_size: 16, + sample_rate: mp4_atom::FixedPoint::new(config.sample_rate as u16, 0), + }; + + match &config.codec { + AudioCodec::AAC(aac) => { + let freq_index: u8 = match config.sample_rate { + 96000 => 0, + 88200 => 1, + 64000 => 2, + 48000 => 3, + 44100 => 4, + 32000 => 5, + 24000 => 6, + 22050 => 7, + 16000 => 8, + 12000 => 9, + 11025 => 10, + 8000 => 11, + 7350 => 12, + _ => 0xF, + }; + + Ok(mp4_atom::Codec::Mp4a(mp4_atom::Mp4a { + audio, + esds: mp4_atom::Esds { + es_desc: mp4_atom::esds::EsDescriptor { + es_id: 1, + dec_config: mp4_atom::esds::DecoderConfig { + object_type_indication: 0x40, + stream_type: 5, + max_bitrate: config.bitrate.unwrap_or(0) as u32, + avg_bitrate: config.bitrate.unwrap_or(0) as u32, + dec_specific: mp4_atom::esds::DecoderSpecific { + profile: aac.profile, + freq_index, + chan_conf: config.channel_count as u8, + }, + ..Default::default() + }, + sl_config: mp4_atom::esds::SLConfig {}, + }, + }, + btrt: None, + taic: None, + })) + } + AudioCodec::Opus => Ok(mp4_atom::Codec::Opus(mp4_atom::Opus { + audio, + dops: mp4_atom::Dops { + output_channel_count: config.channel_count as u8, + pre_skip: 0, + input_sample_rate: config.sample_rate, + output_gain: 0, + }, + btrt: None, + })), + _ => anyhow::bail!("unsupported audio codec for CMAF conversion"), + } +} + +fn guess_video_timescale(config: &VideoConfig) -> u64 { + if let Some(fps) = config.framerate { + (fps * 1000.0) as u64 + } else { + 90000 + } +} diff --git a/rs/moq-mux/src/convert/hang.rs b/rs/moq-mux/src/convert/hang.rs new file mode 100644 index 000000000..e9745203c --- /dev/null +++ b/rs/moq-mux/src/convert/hang.rs @@ -0,0 +1,291 @@ +use anyhow::Context; +use base64::Engine; +use bytes::Bytes; +use hang::catalog::Container; +use hang::container::{Frame, OrderedProducer, Timestamp}; +use mp4_atom::DecodeMaybe; + +/// Converts a broadcast from any format to hang/Legacy format. +/// +/// If tracks are already Legacy, they are passed through unchanged. +/// If tracks are CMAF, parses moof+mdat and converts to hang frames. +pub struct Hang { + input: moq_lite::BroadcastConsumer, + output: moq_lite::BroadcastProducer, +} + +// Make a new audio group every 100ms. +const MAX_AUDIO_GROUP_DURATION: Timestamp = Timestamp::from_millis_unchecked(100); + +impl Hang { + pub fn new(input: moq_lite::BroadcastConsumer, output: moq_lite::BroadcastProducer) -> Self { + Self { input, output } + } + + /// Run the converter. + /// + /// Reads the hang catalog from the input broadcast. If tracks are already Legacy, + /// passes them through unchanged (no-op). If tracks are CMAF, parses moof+mdat + /// and converts to hang frames. + pub async fn run(self) -> anyhow::Result<()> { + let mut broadcast = self.output; + let catalog_producer = crate::CatalogProducer::new(&mut broadcast)?; + + // Subscribe to the input catalog + let catalog_track = self.input.subscribe_track(&hang::Catalog::default_track())?; + let mut catalog_consumer = hang::CatalogConsumer::new(catalog_track); + let catalog = catalog_consumer.next().await?.context("empty catalog")?; + + let mut output_catalog = catalog_producer.clone(); + let mut guard = output_catalog.lock(); + let mut tasks: tokio::task::JoinSet<()> = tokio::task::JoinSet::new(); + + // Convert video tracks + for (name, config) in &catalog.video.renditions { + let input_track = self.input.subscribe_track(&moq_lite::Track { + name: name.clone(), + priority: 1, + })?; + + match &config.container { + Container::Legacy => { + // Already Legacy — pass through + guard.video.renditions.insert(name.clone(), config.clone()); + let output_track = broadcast.create_track(moq_lite::Track { + name: name.clone(), + priority: 1, + })?; + let track_name = name.clone(); + tasks.spawn(async move { + if let Err(e) = passthrough_track(input_track, output_track).await { + tracing::error!(%e, track = %track_name, "passthrough_track failed"); + } + }); + } + Container::Cmaf { init_data } => { + let init_bytes = base64::engine::general_purpose::STANDARD + .decode(init_data) + .context("invalid base64 init_data")?; + + let timescale = parse_timescale(&init_bytes)?; + + let mut legacy_config = config.clone(); + legacy_config.container = Container::Legacy; + guard.video.renditions.insert(name.clone(), legacy_config); + + let output_track = broadcast.create_track(moq_lite::Track { + name: name.clone(), + priority: 1, + })?; + + let track_name = name.clone(); + tasks.spawn(async move { + if let Err(e) = convert_cmaf_to_legacy(input_track, output_track, timescale, true).await { + tracing::error!(%e, track = %track_name, "convert_cmaf_to_legacy failed"); + } + }); + } + } + } + + // Convert audio tracks + for (name, config) in &catalog.audio.renditions { + let input_track = self.input.subscribe_track(&moq_lite::Track { + name: name.clone(), + priority: 2, + })?; + + match &config.container { + Container::Legacy => { + guard.audio.renditions.insert(name.clone(), config.clone()); + let output_track = broadcast.create_track(moq_lite::Track { + name: name.clone(), + priority: 2, + })?; + let track_name = name.clone(); + tasks.spawn(async move { + if let Err(e) = passthrough_track(input_track, output_track).await { + tracing::error!(%e, track = %track_name, "passthrough_track failed"); + } + }); + } + Container::Cmaf { init_data } => { + let init_bytes = base64::engine::general_purpose::STANDARD + .decode(init_data) + .context("invalid base64 init_data")?; + + let timescale = parse_timescale(&init_bytes)?; + + let mut legacy_config = config.clone(); + legacy_config.container = Container::Legacy; + guard.audio.renditions.insert(name.clone(), legacy_config); + + let output_track = broadcast.create_track(moq_lite::Track { + name: name.clone(), + priority: 2, + })?; + + let track_name = name.clone(); + tasks.spawn(async move { + if let Err(e) = convert_cmaf_to_legacy(input_track, output_track, timescale, false).await { + tracing::error!(%e, track = %track_name, "convert_cmaf_to_legacy failed"); + } + }); + } + } + } + + drop(guard); + + // Keep broadcast and catalog alive until all track tasks complete. + while tasks.join_next().await.is_some() {} + + Ok(()) + } +} + +/// Parse the timescale from an init segment (ftyp+moov). +fn parse_timescale(init_data: &[u8]) -> anyhow::Result { + let mut cursor = std::io::Cursor::new(init_data); + while let Some(atom) = mp4_atom::Any::decode_maybe(&mut cursor)? { + if let mp4_atom::Any::Moov(moov) = atom { + let trak = moov.trak.first().context("no tracks in moov")?; + return Ok(trak.mdia.mdhd.timescale as u64); + } + } + anyhow::bail!("no moov found in init data") +} + +/// Pass a track through unchanged. +async fn passthrough_track( + mut input: moq_lite::TrackConsumer, + mut output: moq_lite::TrackProducer, +) -> anyhow::Result<()> { + while let Some(group) = input.recv_group().await? { + let mut out_group = output.append_group()?; + let mut frame_reader = group; + while let Some(frame_data) = frame_reader.read_frame().await? { + out_group.write_frame(frame_data)?; + } + out_group.finish()?; + } + output.finish()?; + Ok(()) +} + +/// Convert CMAF moof+mdat frames to hang Legacy frames. +async fn convert_cmaf_to_legacy( + mut input: moq_lite::TrackConsumer, + output: moq_lite::TrackProducer, + timescale: u64, + is_video: bool, +) -> anyhow::Result<()> { + let mut ordered = OrderedProducer::new(output); + + if !is_video { + ordered = ordered.with_max_group_duration(MAX_AUDIO_GROUP_DURATION); + } + + while let Some(group) = input.recv_group().await? { + let mut frame_reader = group; + let mut is_first_in_group = true; + + while let Some(frame_data) = frame_reader.read_frame().await? { + // Parse the moof+mdat fragment + let samples = extract_samples(&frame_data, timescale)?; + + for (i, (timestamp, payload, keyframe)) in samples.into_iter().enumerate() { + if is_video && is_first_in_group && i == 0 && keyframe { + ordered.keyframe()?; + } + + let frame = Frame { + timestamp, + payload: payload.into(), + }; + ordered.write(frame)?; + } + + is_first_in_group = false; + } + } + + ordered.finish()?; + Ok(()) +} + +/// Extract individual samples from a moof+mdat fragment. +fn extract_samples(data: &Bytes, timescale: u64) -> anyhow::Result> { + let mut cursor = std::io::Cursor::new(data.as_ref()); + let mut moof: Option = None; + + while let Some(atom) = mp4_atom::Any::decode_maybe(&mut cursor)? { + match atom { + mp4_atom::Any::Moof(m) => { + moof = Some(m); + } + mp4_atom::Any::Mdat(mdat) => { + let moof = moof.take().context("mdat without moof")?; + return extract_from_moof_mdat(&moof, &mdat, timescale); + } + _ => {} + } + } + + anyhow::bail!("no mdat found in fragment") +} + +fn extract_from_moof_mdat( + moof: &mp4_atom::Moof, + mdat: &mp4_atom::Mdat, + timescale: u64, +) -> anyhow::Result> { + let mut samples = Vec::new(); + + for traf in &moof.traf { + let tfdt = traf.tfdt.as_ref().context("missing tfdt")?; + let mut dts = tfdt.base_media_decode_time; + let mut offset = 0usize; + + for trun in &traf.trun { + if trun.data_offset.is_some() { + // data_offset is relative to start of moof. Since we converted the + // fragment ourselves (build_moof_mdat sets data_offset = moof_size + 8), + // we subtract those to get an offset into mdat.data. + // For fragments we produce, data_offset points past the mdat header, + // so the offset into mdat.data is 0 for the first sample. + // For external fragments we don't have moof_size, so we reset to 0. + offset = 0; + } + + for entry in &trun.entries { + let flags = entry.flags.unwrap_or(traf.tfhd.default_sample_flags.unwrap_or(0)); + let duration = entry.duration.unwrap_or(traf.tfhd.default_sample_duration.unwrap_or(0)); + let size = entry.size.unwrap_or(traf.tfhd.default_sample_size.unwrap_or(0)) as usize; + + let pts = (dts as i64 + entry.cts.unwrap_or_default() as i64) as u64; + let timestamp = Timestamp::from_scale(pts, timescale)?; + + let keyframe = { + let depends_on_no_other = (flags >> 24) & 0x3 == 0x2; + let non_sync = (flags >> 16) & 0x1 == 0x1; + depends_on_no_other && !non_sync + }; + + anyhow::ensure!( + offset + size <= mdat.data.len(), + "sample extends past mdat: offset={offset} size={size} mdat_len={}", + mdat.data.len() + ); + + let payload = Bytes::copy_from_slice(&mdat.data[offset..offset + size]); + samples.push((timestamp, payload, keyframe)); + + dts += duration as u64; + offset += size; + } + } + } + + Ok(samples) +} diff --git a/rs/moq-mux/src/convert/mod.rs b/rs/moq-mux/src/convert/mod.rs new file mode 100644 index 000000000..e582408f8 --- /dev/null +++ b/rs/moq-mux/src/convert/mod.rs @@ -0,0 +1,8 @@ +mod fmp4; +mod hang; + +pub use self::hang::*; +pub use fmp4::*; + +#[cfg(test)] +mod test; diff --git a/rs/moq-mux/src/convert/test.rs b/rs/moq-mux/src/convert/test.rs new file mode 100644 index 000000000..1582c978d --- /dev/null +++ b/rs/moq-mux/src/convert/test.rs @@ -0,0 +1,418 @@ +use std::time::Duration; + +use base64::Engine; +use buf_list::BufList; +use bytes::{Bytes, BytesMut}; +use hang::catalog::{Container, H264, VideoCodec, VideoConfig}; +use hang::container::{Frame, Timestamp}; +use mp4_atom::{Atom, DecodeMaybe}; + +use super::fmp4::{build_moof_mdat, build_video_init}; + +// ---- Helpers ---- + +fn ts(micros: u64) -> Timestamp { + Timestamp::from_micros(micros).unwrap() +} + +fn build_avcc_description() -> Bytes { + let avcc = mp4_atom::Avcc { + configuration_version: 1, + avc_profile_indication: 0x64, + profile_compatibility: 0x00, + avc_level_indication: 0x1F, + length_size: 4, + sequence_parameter_sets: vec![vec![ + 0x67, 0x64, 0x00, 0x1F, 0xAC, 0xD9, 0x40, 0x50, 0x05, 0xBB, 0x01, 0x10, 0x00, 0x00, 0x03, 0x00, 0x10, 0x00, + 0x00, 0x03, 0x03, 0xC0, 0xF1, 0x62, 0xE4, 0x80, + ]], + picture_parameter_sets: vec![vec![0x68, 0xEB, 0xE3, 0xCB, 0x22, 0xC0]], + ..Default::default() + }; + + let mut buf = BytesMut::new(); + avcc.encode_body(&mut buf).expect("encode avcc"); + buf.freeze() +} + +fn test_video_config() -> VideoConfig { + VideoConfig { + codec: VideoCodec::H264(H264 { + profile: 0x64, + constraints: 0x00, + level: 0x1F, + inline: false, + }), + description: Some(build_avcc_description()), + coded_width: Some(1920), + coded_height: Some(1080), + framerate: Some(30.0), + container: Container::Legacy, + bitrate: None, + display_ratio_width: None, + display_ratio_height: None, + optimize_for_latency: None, + jitter: None, + } +} + +/// Set up an input broadcast with the given catalog config. +fn setup_input( + video_config: &VideoConfig, +) -> ( + moq_lite::BroadcastConsumer, + moq_lite::TrackProducer, + moq_lite::BroadcastProducer, + moq_lite::TrackProducer, +) { + let mut broadcast = moq_lite::Broadcast::new().produce(); + + let mut catalog_track = broadcast.create_track(hang::Catalog::default_track()).unwrap(); + let mut catalog = hang::Catalog::default(); + catalog + .video + .renditions + .insert("video".to_string(), video_config.clone()); + + let catalog_json = catalog.to_string().unwrap(); + let mut group = catalog_track.append_group().unwrap(); + group.write_frame(catalog_json).unwrap(); + group.finish().unwrap(); + + let video_track = broadcast + .create_track(moq_lite::Track { + name: "video".to_string(), + priority: 1, + }) + .unwrap(); + + let consumer = broadcast.consume(); + + (consumer, video_track, broadcast, catalog_track) +} + +fn write_legacy_frames(track: &mut moq_lite::TrackProducer, frames: &[(Timestamp, Vec, bool)]) { + let mut current_group: Option = None; + for (timestamp, payload, is_keyframe) in frames { + if *is_keyframe { + if let Some(mut g) = current_group.take() { + g.finish().unwrap(); + } + current_group = Some(track.append_group().unwrap()); + } else if current_group.is_none() { + current_group = Some(track.append_group().unwrap()); + } + + let frame = Frame { + timestamp: *timestamp, + payload: BufList::from_iter(vec![Bytes::from(payload.clone())]), + }; + frame.encode(current_group.as_mut().unwrap()).unwrap(); + } + + if let Some(mut g) = current_group.take() { + g.finish().unwrap(); + } + track.finish().unwrap(); +} + +fn write_cmaf_frames(track: &mut moq_lite::TrackProducer, frames: &[(u64, Vec, bool)]) { + let mut current_group: Option = None; + let mut seq: u32 = 1; + for (dts, payload, keyframe) in frames { + if *keyframe { + if let Some(mut g) = current_group.take() { + g.finish().unwrap(); + } + current_group = Some(track.append_group().unwrap()); + } else if current_group.is_none() { + current_group = Some(track.append_group().unwrap()); + } + + let moof_mdat = build_moof_mdat(seq, 1, *dts, payload, *keyframe).unwrap(); + seq += 1; + current_group.as_mut().unwrap().write_frame(moof_mdat).unwrap(); + } + + if let Some(mut g) = current_group.take() { + g.finish().unwrap(); + } + track.finish().unwrap(); +} + +/// Read all Legacy frames from a track consumer (must be subscribed before converter finishes). +async fn read_legacy_frames(track: moq_lite::TrackConsumer) -> Vec<(Timestamp, Vec, bool)> { + let mut ordered = crate::consumer::OrderedConsumer::new(track, crate::consumer::Legacy, Duration::MAX); + + let mut result = Vec::new(); + while let Some(frame) = tokio::time::timeout(Duration::from_millis(500), ordered.read()) + .await + .expect("read_legacy_frames timed out") + .expect("read_legacy_frames error") + { + let is_keyframe = frame.is_keyframe(); + let timestamp = frame.timestamp; + let payload: Vec = frame.payload.into_iter().flat_map(|c| c.into_iter()).collect(); + result.push((timestamp, payload, is_keyframe)); + } + result +} + +/// Read all raw CMAF frames from a track consumer (must be subscribed before converter finishes). +async fn read_cmaf_raw_frames(mut track: moq_lite::TrackConsumer) -> Vec { + let mut result = Vec::new(); + while let Some(group) = tokio::time::timeout(Duration::from_millis(500), track.recv_group()) + .await + .expect("read_cmaf_raw_frames timed out") + .expect("read_cmaf_raw_frames error") + { + let mut reader = group; + while let Some(data) = tokio::time::timeout(Duration::from_millis(500), reader.read_frame()) + .await + .expect("read_cmaf_raw_frames timed out on frame") + .expect("read_cmaf_raw_frames frame error") + { + result.push(data); + } + } + result +} + +fn parse_cmaf_frame(data: &Bytes, timescale: u64) -> (Timestamp, Vec, bool) { + let mut cursor = std::io::Cursor::new(data.as_ref()); + let mut moof_found = None; + let mut mdat_found = None; + + while let Some(atom) = mp4_atom::Any::decode_maybe(&mut cursor).unwrap() { + match atom { + mp4_atom::Any::Moof(m) => moof_found = Some(m), + mp4_atom::Any::Mdat(m) => mdat_found = Some(m), + _ => {} + } + } + + let moof = moof_found.expect("no moof"); + let mdat = mdat_found.expect("no mdat"); + let traf = &moof.traf[0]; + let tfdt = traf.tfdt.as_ref().expect("no tfdt"); + let timestamp = Timestamp::from_scale(tfdt.base_media_decode_time, timescale).unwrap(); + let flags = traf.trun[0].entries[0].flags.unwrap_or(0); + let keyframe = (flags >> 24) & 0x3 == 0x2 && (flags >> 16) & 0x1 == 0; + + (timestamp, mdat.data.clone(), keyframe) +} + +/// Subscribe to the video track, retrying until it appears. +async fn subscribe_video(consumer: &moq_lite::BroadcastConsumer) -> moq_lite::TrackConsumer { + let track = moq_lite::Track { + name: "video".to_string(), + priority: 1, + }; + loop { + match consumer.subscribe_track(&track) { + Ok(t) => return t, + Err(_) => tokio::task::yield_now().await, + } + } +} + +// ---- Tests ---- + +#[tokio::test] +async fn legacy_to_cmaf_video() { + let config = test_video_config(); + let frames = vec![ + (ts(0), vec![0x01, 0x02, 0x03], true), + (ts(33_000), vec![0x04, 0x05], false), + (ts(66_000), vec![0x06, 0x07, 0x08], true), + ]; + + let (consumer, mut video_track, _broadcast, _catalog_track) = setup_input(&config); + let output = moq_lite::Broadcast::new().produce(); + let output_consumer = output.consume(); + + let converter = super::Fmp4::new(consumer, output); + + let frames_clone = frames.clone(); + tokio::spawn(async move { + tokio::task::yield_now().await; + write_legacy_frames(&mut video_track, &frames_clone); + }); + + let (convert_result, cmaf_frames) = tokio::join!(converter.run(), async { + let output_video = subscribe_video(&output_consumer).await; + read_cmaf_raw_frames(output_video).await + }); + convert_result.unwrap(); + + let timescale = config.framerate.map(|f| (f * 1000.0) as u64).unwrap(); + assert_eq!(cmaf_frames.len(), 3, "expected 3 CMAF frames"); + + for (i, cmaf_data) in cmaf_frames.iter().enumerate() { + let (parsed_ts, payload, keyframe) = parse_cmaf_frame(cmaf_data, timescale); + assert_eq!(parsed_ts, frames[i].0, "timestamp mismatch at frame {i}"); + assert_eq!(payload, frames[i].1, "payload mismatch at frame {i}"); + assert_eq!(keyframe, frames[i].2, "keyframe flag mismatch at frame {i}"); + } +} + +#[tokio::test] +async fn cmaf_to_legacy_video() { + let config = test_video_config(); + let init_data = build_video_init(&config).unwrap(); + let timescale = config.framerate.map(|f| (f * 1000.0) as u64).unwrap(); + + let cmaf_frames: Vec<(u64, Vec, bool)> = vec![ + (0, vec![0x01, 0x02, 0x03], true), + (33_000u64 * timescale / 1_000_000, vec![0x04, 0x05], false), + (66_000u64 * timescale / 1_000_000, vec![0x06, 0x07, 0x08], true), + ]; + + let mut cmaf_config = config.clone(); + cmaf_config.container = Container::Cmaf { + init_data: base64::engine::general_purpose::STANDARD.encode(&init_data), + }; + + let (consumer, mut video_track, _broadcast, _catalog_track) = setup_input(&cmaf_config); + let output = moq_lite::Broadcast::new().produce(); + let output_consumer = output.consume(); + let converter = super::Hang::new(consumer, output); + + let cmaf_frames_clone = cmaf_frames.clone(); + tokio::spawn(async move { + tokio::task::yield_now().await; + write_cmaf_frames(&mut video_track, &cmaf_frames_clone); + }); + + let (convert_result, legacy_frames) = tokio::join!(converter.run(), async { + let output_video = subscribe_video(&output_consumer).await; + read_legacy_frames(output_video).await + }); + convert_result.unwrap(); + + assert_eq!(legacy_frames.len(), 3, "expected 3 Legacy frames"); + assert_eq!(legacy_frames[0].0, ts(0)); + assert_eq!(legacy_frames[0].1, vec![0x01, 0x02, 0x03]); + assert_eq!(legacy_frames[1].0, ts(33_000)); + assert_eq!(legacy_frames[1].1, vec![0x04, 0x05]); + assert_eq!(legacy_frames[2].0, ts(66_000)); + assert_eq!(legacy_frames[2].1, vec![0x06, 0x07, 0x08]); +} + +#[tokio::test] +async fn roundtrip_legacy_cmaf_legacy() { + let config = test_video_config(); + let frames = vec![ + (ts(0), vec![0xAA, 0xBB], true), + (ts(33_000), vec![0xCC], false), + (ts(66_000), vec![0xDD, 0xEE, 0xFF], true), + (ts(99_000), vec![0x11, 0x22], false), + ]; + + let (consumer, mut video_track, _broadcast, _catalog_track) = setup_input(&config); + + // Legacy → CMAF + let cmaf_output = moq_lite::Broadcast::new().produce(); + let cmaf_consumer = cmaf_output.consume(); + let fmp4_converter = super::Fmp4::new(consumer, cmaf_output); + + // CMAF → Legacy + let legacy_output = moq_lite::Broadcast::new().produce(); + let legacy_consumer = legacy_output.consume(); + let hang_converter = super::Hang::new(cmaf_consumer, legacy_output); + + let frames_clone = frames.clone(); + tokio::spawn(async move { + tokio::task::yield_now().await; + write_legacy_frames(&mut video_track, &frames_clone); + }); + + let (r1, r2, result) = tokio::join!(fmp4_converter.run(), hang_converter.run(), async { + let legacy_video = subscribe_video(&legacy_consumer).await; + read_legacy_frames(legacy_video).await + }); + r1.unwrap(); + r2.unwrap(); + + assert_eq!(result.len(), frames.len(), "frame count mismatch after roundtrip"); + + for (i, (expected_ts, expected_payload, _)) in frames.iter().enumerate() { + assert_eq!(result[i].0, *expected_ts, "timestamp mismatch at frame {i}"); + assert_eq!(result[i].1, *expected_payload, "payload mismatch at frame {i}"); + } +} + +#[tokio::test] +async fn cmaf_passthrough() { + let config = test_video_config(); + let init_data = build_video_init(&config).unwrap(); + let timescale = config.framerate.map(|f| (f * 1000.0) as u64).unwrap(); + + let cmaf_frames: Vec<(u64, Vec, bool)> = vec![ + (0, vec![0x01, 0x02], true), + (33_000u64 * timescale / 1_000_000, vec![0x03, 0x04], false), + ]; + + let mut cmaf_config = config.clone(); + cmaf_config.container = Container::Cmaf { + init_data: base64::engine::general_purpose::STANDARD.encode(&init_data), + }; + + let (consumer, mut video_track, _broadcast, _catalog_track) = setup_input(&cmaf_config); + let output = moq_lite::Broadcast::new().produce(); + let output_consumer = output.consume(); + + let converter = super::Fmp4::new(consumer, output); + + let cmaf_frames_clone = cmaf_frames.clone(); + tokio::spawn(async move { + tokio::task::yield_now().await; + write_cmaf_frames(&mut video_track, &cmaf_frames_clone); + }); + + let (convert_result, output_frames) = tokio::join!(converter.run(), async { + let output_video = subscribe_video(&output_consumer).await; + read_cmaf_raw_frames(output_video).await + }); + convert_result.unwrap(); + + assert_eq!(output_frames.len(), cmaf_frames.len()); + + let mut seq = 1u32; + for (i, (dts, payload, keyframe)) in cmaf_frames.iter().enumerate() { + let expected = build_moof_mdat(seq, 1, *dts, payload, *keyframe).unwrap(); + seq += 1; + assert_eq!(output_frames[i], expected, "frame {i} should be byte-identical"); + } +} + +#[tokio::test] +async fn legacy_passthrough() { + let config = test_video_config(); + let frames = vec![(ts(0), vec![0xAA, 0xBB], true), (ts(33_000), vec![0xCC, 0xDD], false)]; + + let (consumer, mut video_track, _broadcast, _catalog_track) = setup_input(&config); + let output = moq_lite::Broadcast::new().produce(); + let output_consumer = output.consume(); + + let converter = super::Hang::new(consumer, output); + + let frames_clone = frames.clone(); + tokio::spawn(async move { + tokio::task::yield_now().await; + write_legacy_frames(&mut video_track, &frames_clone); + }); + + let (convert_result, result) = tokio::join!(converter.run(), async { + let output_video = subscribe_video(&output_consumer).await; + read_legacy_frames(output_video).await + }); + convert_result.expect("converter.run() failed"); + + assert_eq!(result.len(), frames.len()); + + for (i, (expected_ts, expected_payload, _)) in frames.iter().enumerate() { + assert_eq!(result[i].0, *expected_ts, "timestamp mismatch at frame {i}"); + assert_eq!(result[i].1, *expected_payload, "payload mismatch at frame {i}"); + } +} diff --git a/rs/moq-mux/src/lib.rs b/rs/moq-mux/src/lib.rs index 501488931..143076299 100644 --- a/rs/moq-mux/src/lib.rs +++ b/rs/moq-mux/src/lib.rs @@ -1,7 +1,11 @@ -//! Media demuxers and (soon) muxers for MoQ. +//! Media muxers and demuxers for MoQ. mod catalog; -pub mod import; +#[cfg(feature = "mp4")] +pub mod consumer; +#[cfg(feature = "mp4")] +pub mod convert; pub mod msf; +pub mod producer; pub use catalog::*; diff --git a/rs/moq-mux/src/import/aac.rs b/rs/moq-mux/src/producer/aac.rs similarity index 100% rename from rs/moq-mux/src/import/aac.rs rename to rs/moq-mux/src/producer/aac.rs diff --git a/rs/moq-mux/src/import/annexb.rs b/rs/moq-mux/src/producer/annexb.rs similarity index 100% rename from rs/moq-mux/src/import/annexb.rs rename to rs/moq-mux/src/producer/annexb.rs diff --git a/rs/moq-mux/src/import/av01.rs b/rs/moq-mux/src/producer/av01.rs similarity index 100% rename from rs/moq-mux/src/import/av01.rs rename to rs/moq-mux/src/producer/av01.rs diff --git a/rs/moq-mux/src/import/avc3.rs b/rs/moq-mux/src/producer/avc3.rs similarity index 100% rename from rs/moq-mux/src/import/avc3.rs rename to rs/moq-mux/src/producer/avc3.rs diff --git a/rs/moq-mux/src/import/decoder.rs b/rs/moq-mux/src/producer/decoder.rs similarity index 79% rename from rs/moq-mux/src/import/decoder.rs rename to rs/moq-mux/src/producer/decoder.rs index d5db4f74f..59eb5c1b7 100644 --- a/rs/moq-mux/src/import/decoder.rs +++ b/rs/moq-mux/src/producer/decoder.rs @@ -3,10 +3,10 @@ use std::{fmt, str::FromStr}; use bytes::Buf; use hang::Error; -/// The supported decoder formats. +/// The supported framed formats (known frame boundaries). #[derive(Clone, Copy, Debug, Eq, PartialEq)] #[non_exhaustive] -pub enum DecoderFormat { +pub enum FramedFormat { /// aka H264 with inline SPS/PPS #[cfg(feature = "h264")] Avc3, @@ -27,48 +27,51 @@ pub enum DecoderFormat { Opus, } -impl FromStr for DecoderFormat { +#[deprecated(note = "use FramedFormat instead")] +pub type DecoderFormat = FramedFormat; + +impl FromStr for FramedFormat { type Err = Error; fn from_str(s: &str) -> Result { match s { #[cfg(feature = "h264")] - "avc3" => Ok(DecoderFormat::Avc3), + "avc3" => Ok(FramedFormat::Avc3), #[cfg(feature = "h264")] "h264" | "annex-b" => { tracing::warn!("format '{s}' is deprecated, use 'avc3' instead"); - Ok(DecoderFormat::Avc3) + Ok(FramedFormat::Avc3) } #[cfg(feature = "h265")] - "hev1" => Ok(DecoderFormat::Hev1), + "hev1" => Ok(FramedFormat::Hev1), #[cfg(feature = "mp4")] - "fmp4" | "cmaf" => Ok(DecoderFormat::Fmp4), + "fmp4" | "cmaf" => Ok(FramedFormat::Fmp4), #[cfg(feature = "av1")] - "av01" | "av1" | "av1C" => Ok(DecoderFormat::Av01), + "av01" | "av1" | "av1C" => Ok(FramedFormat::Av01), #[cfg(feature = "aac")] - "aac" => Ok(DecoderFormat::Aac), + "aac" => Ok(FramedFormat::Aac), #[cfg(feature = "opus")] - "opus" => Ok(DecoderFormat::Opus), + "opus" => Ok(FramedFormat::Opus), _ => Err(Error::UnknownFormat(s.to_string())), } } } -impl fmt::Display for DecoderFormat { +impl fmt::Display for FramedFormat { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match *self { #[cfg(feature = "h264")] - DecoderFormat::Avc3 => write!(f, "avc3"), + FramedFormat::Avc3 => write!(f, "avc3"), #[cfg(feature = "mp4")] - DecoderFormat::Fmp4 => write!(f, "fmp4"), + FramedFormat::Fmp4 => write!(f, "fmp4"), #[cfg(feature = "h265")] - DecoderFormat::Hev1 => write!(f, "hev1"), + FramedFormat::Hev1 => write!(f, "hev1"), #[cfg(feature = "av1")] - DecoderFormat::Av01 => write!(f, "av01"), + FramedFormat::Av01 => write!(f, "av01"), #[cfg(feature = "aac")] - DecoderFormat::Aac => write!(f, "aac"), + FramedFormat::Aac => write!(f, "aac"), #[cfg(feature = "opus")] - DecoderFormat::Opus => write!(f, "opus"), + FramedFormat::Opus => write!(f, "opus"), } } } @@ -129,17 +132,17 @@ impl fmt::Display for StreamFormat { } } -impl From for DecoderFormat { +impl From for FramedFormat { fn from(format: StreamFormat) -> Self { match format { #[cfg(feature = "h264")] - StreamFormat::Avc3 => DecoderFormat::Avc3, + StreamFormat::Avc3 => FramedFormat::Avc3, #[cfg(feature = "mp4")] - StreamFormat::Fmp4 => DecoderFormat::Fmp4, + StreamFormat::Fmp4 => FramedFormat::Fmp4, #[cfg(feature = "h265")] - StreamFormat::Hev1 => DecoderFormat::Hev1, + StreamFormat::Hev1 => FramedFormat::Hev1, #[cfg(feature = "av1")] - StreamFormat::Av01 => DecoderFormat::Av01, + StreamFormat::Av01 => FramedFormat::Av01, } } } @@ -160,7 +163,7 @@ enum StreamKind { } #[derive(derive_more::From)] -enum DecoderKind { +enum FramedKind { /// aka H264 with inline SPS/PPS #[cfg(feature = "h264")] Avc3(super::Avc3), @@ -178,22 +181,25 @@ enum DecoderKind { Opus(super::Opus), } -/// A decoder for formats that support stream decoding (unknown frame boundaries). +/// An importer for formats that support stream decoding (unknown frame boundaries). /// /// This includes formats like H.264 (AVC3), H.265 (HEV1), and fMP4/CMAF. /// Use this when the caller does not know the frame boundaries. -pub struct StreamDecoder { +pub struct Stream { decoder: StreamKind, } -impl StreamDecoder { - /// Create a new stream decoder with the given format. +#[deprecated(note = "use Stream instead")] +pub type StreamDecoder = Stream; + +impl Stream { + /// Create a new stream importer with the given format. pub fn new(broadcast: moq_lite::BroadcastProducer, catalog: crate::CatalogProducer, format: StreamFormat) -> Self { let decoder = match format { #[cfg(feature = "h264")] StreamFormat::Avc3 => super::Avc3::new(broadcast, catalog).into(), #[cfg(feature = "mp4")] - StreamFormat::Fmp4 => Box::new(super::Fmp4::new(broadcast, catalog, super::Fmp4Config::default())).into(), + StreamFormat::Fmp4 => Box::new(super::Fmp4::new(broadcast, catalog)).into(), #[cfg(feature = "h265")] StreamFormat::Hev1 => super::Hev1::new(broadcast, catalog).into(), #[cfg(feature = "av1")] @@ -279,55 +285,58 @@ impl StreamDecoder { } } -/// A decoder for formats with known frame boundaries. +/// An importer for formats with known frame boundaries. /// /// This supports all formats and should be used when the caller knows the frame boundaries. -pub struct Decoder { - decoder: DecoderKind, +pub struct Framed { + decoder: FramedKind, } -impl Decoder { - /// Create a new decoder with the given format and initialization data. +#[deprecated(note = "use Framed instead")] +pub type Decoder = Framed; + +impl Framed { + /// Create a new framed importer with the given format and initialization data. /// /// The buffer will be fully consumed, or an error will be returned. pub fn new>( broadcast: moq_lite::BroadcastProducer, catalog: crate::CatalogProducer, - format: DecoderFormat, + format: FramedFormat, buf: &mut T, ) -> anyhow::Result { let decoder = match format { #[cfg(feature = "h264")] - DecoderFormat::Avc3 => { + FramedFormat::Avc3 => { let mut decoder = super::Avc3::new(broadcast, catalog); decoder.initialize(buf)?; decoder.into() } #[cfg(feature = "mp4")] - DecoderFormat::Fmp4 => { - let mut decoder = Box::new(super::Fmp4::new(broadcast, catalog, super::Fmp4Config::default())); + FramedFormat::Fmp4 => { + let mut decoder = Box::new(super::Fmp4::new(broadcast, catalog)); decoder.decode(buf)?; decoder.into() } #[cfg(feature = "h265")] - DecoderFormat::Hev1 => { + FramedFormat::Hev1 => { let mut decoder = super::Hev1::new(broadcast, catalog); decoder.initialize(buf)?; decoder.into() } #[cfg(feature = "av1")] - DecoderFormat::Av01 => { + FramedFormat::Av01 => { let mut decoder = super::Av01::new(broadcast, catalog); decoder.initialize(buf)?; decoder.into() } #[cfg(feature = "aac")] - DecoderFormat::Aac => { + FramedFormat::Aac => { let config = super::AacConfig::parse(buf)?; super::Aac::new(broadcast, catalog, config)?.into() } #[cfg(feature = "opus")] - DecoderFormat::Opus => { + FramedFormat::Opus => { let config = super::OpusConfig::parse(buf)?; super::Opus::new(broadcast, catalog, config)?.into() } @@ -345,17 +354,17 @@ impl Decoder { pub fn finish(&mut self) -> anyhow::Result<()> { match self.decoder { #[cfg(feature = "h264")] - DecoderKind::Avc3(ref mut decoder) => decoder.finish(), + FramedKind::Avc3(ref mut decoder) => decoder.finish(), #[cfg(feature = "mp4")] - DecoderKind::Fmp4(ref mut decoder) => decoder.finish(), + FramedKind::Fmp4(ref mut decoder) => decoder.finish(), #[cfg(feature = "h265")] - DecoderKind::Hev1(ref mut decoder) => decoder.finish(), + FramedKind::Hev1(ref mut decoder) => decoder.finish(), #[cfg(feature = "av1")] - DecoderKind::Av01(ref mut decoder) => decoder.finish(), + FramedKind::Av01(ref mut decoder) => decoder.finish(), #[cfg(feature = "aac")] - DecoderKind::Aac(ref mut decoder) => decoder.finish(), + FramedKind::Aac(ref mut decoder) => decoder.finish(), #[cfg(feature = "opus")] - DecoderKind::Opus(ref mut decoder) => decoder.finish(), + FramedKind::Opus(ref mut decoder) => decoder.finish(), } } @@ -375,17 +384,17 @@ impl Decoder { ) -> anyhow::Result<()> { match self.decoder { #[cfg(feature = "h264")] - DecoderKind::Avc3(ref mut decoder) => decoder.decode_frame(buf, pts)?, + FramedKind::Avc3(ref mut decoder) => decoder.decode_frame(buf, pts)?, #[cfg(feature = "mp4")] - DecoderKind::Fmp4(ref mut decoder) => decoder.decode(buf)?, + FramedKind::Fmp4(ref mut decoder) => decoder.decode(buf)?, #[cfg(feature = "h265")] - DecoderKind::Hev1(ref mut decoder) => decoder.decode_frame(buf, pts)?, + FramedKind::Hev1(ref mut decoder) => decoder.decode_frame(buf, pts)?, #[cfg(feature = "av1")] - DecoderKind::Av01(ref mut decoder) => decoder.decode_frame(buf, pts)?, + FramedKind::Av01(ref mut decoder) => decoder.decode_frame(buf, pts)?, #[cfg(feature = "aac")] - DecoderKind::Aac(ref mut decoder) => decoder.decode(buf, pts)?, + FramedKind::Aac(ref mut decoder) => decoder.decode(buf, pts)?, #[cfg(feature = "opus")] - DecoderKind::Opus(ref mut decoder) => decoder.decode(buf, pts)?, + FramedKind::Opus(ref mut decoder) => decoder.decode(buf, pts)?, } anyhow::ensure!(!buf.has_remaining(), "buffer was not fully consumed"); @@ -395,14 +404,14 @@ impl Decoder { } #[cfg(feature = "opus")] -impl From for Decoder { +impl From for Framed { fn from(opus: super::Opus) -> Self { Self { decoder: opus.into() } } } #[cfg(feature = "aac")] -impl From for Decoder { +impl From for Framed { fn from(aac: super::Aac) -> Self { Self { decoder: aac.into() } } diff --git a/rs/moq-mux/src/import/fmp4.rs b/rs/moq-mux/src/producer/fmp4.rs similarity index 77% rename from rs/moq-mux/src/import/fmp4.rs rename to rs/moq-mux/src/producer/fmp4.rs index 13ff56bc8..56c992ba6 100644 --- a/rs/moq-mux/src/import/fmp4.rs +++ b/rs/moq-mux/src/producer/fmp4.rs @@ -7,19 +7,10 @@ use mp4_atom::{Any, Atom, DecodeMaybe, Encode, Mdat, Moof, Moov, Trak}; use std::collections::HashMap; use tokio::io::{AsyncRead, AsyncReadExt}; -/// Configuration for the fMP4 importer. -#[derive(Clone, Default)] -pub struct Fmp4Config { - /// When true, transport fMP4 fragments directly (passthrough mode) - /// - /// This requires a player that can decode the fragments directly. - pub passthrough: bool, -} - -/// Converts fMP4/CMAF files into hang broadcast streams. +/// Converts fMP4/CMAF files into MoQ broadcast streams using CMAF passthrough. /// -/// This struct processes fragmented MP4 (fMP4) files and converts them into hang broadcasts. -/// Not all MP4 features are supported. +/// This struct processes fragmented MP4 (fMP4) files and transports complete +/// moof+mdat fragments directly as MoQ frames, preserving the CMAF container format. /// /// ## Supported Codecs /// @@ -50,10 +41,7 @@ pub struct Fmp4 { moof: Option, moof_size: usize, - /// Configuration for the fMP4 importer. - config: Fmp4Config, - - // -- PASSTHROUGH ONLY -- + // The raw moof bytes for passthrough moof_raw: Option, } @@ -63,32 +51,11 @@ enum TrackKind { Audio, } -// Make a new audio group every 100ms. -const MAX_AUDIO_GROUP_DURATION: Timestamp = Timestamp::from_millis_unchecked(100); - -enum Fmp4Producer { - /// Manual group management (video, passthrough audio). - Manual { - track: moq_lite::TrackProducer, - group: Option, - }, - /// Automatic duration-based group management (non-passthrough audio). - Ordered(hang::container::OrderedProducer), -} - -impl Fmp4Producer { - fn info(&self) -> &moq_lite::Track { - match self { - Self::Manual { track, .. } => &track.info, - Self::Ordered(ordered) => &ordered.track.info, - } - } -} - struct Fmp4Track { kind: TrackKind, - producer: Fmp4Producer, + track: moq_lite::TrackProducer, + group: Option, // The minimum buffer required for the track. jitter: Option, @@ -104,7 +71,7 @@ impl Fmp4 { /// Create a new CMAF importer that will write to the given broadcast. /// /// The broadcast will be populated with tracks as they're discovered in the fMP4 file. - pub fn new(broadcast: moq_lite::BroadcastProducer, catalog: crate::CatalogProducer, config: Fmp4Config) -> Self { + pub fn new(broadcast: moq_lite::BroadcastProducer, catalog: crate::CatalogProducer) -> Self { Self { catalog, tracks: HashMap::default(), @@ -112,7 +79,6 @@ impl Fmp4 { moof: None, moof_size: 0, broadcast, - config, moof_raw: None, } } @@ -148,10 +114,7 @@ impl Fmp4 { anyhow::ensure!(self.moof.is_none(), "duplicate moof box"); self.moof.replace(moof); self.moof_size = size; - - if self.config.passthrough { - self.moof_raw.replace(Bytes::copy_from_slice(raw)); - } + self.moof_raw.replace(Bytes::copy_from_slice(raw)); } Any::Mdat(mdat) => { self.extract(mdat, raw)?; @@ -183,7 +146,7 @@ impl Fmp4 { for trak in &moov.trak { let track_id = trak.tkhd.track_id; let handler = &trak.mdia.hdlr.handler; - let ext = if self.config.passthrough { "m4s" } else { "hang" }; + let ext = "m4s"; let (kind, track) = match handler.as_ref() { b"vide" => { @@ -202,19 +165,12 @@ impl Fmp4 { let track = self.broadcast.create_track(track)?; - let producer = if kind == TrackKind::Audio && !self.config.passthrough { - Fmp4Producer::Ordered( - hang::container::OrderedProducer::new(track).with_max_group_duration(MAX_AUDIO_GROUP_DURATION), - ) - } else { - Fmp4Producer::Manual { track, group: None } - }; - self.tracks.insert( track_id, Fmp4Track { kind, - producer, + track, + group: None, jitter: None, last_timestamp: None, min_duration: None, @@ -230,8 +186,8 @@ impl Fmp4 { } fn container(&self, trak: &Trak, moov: &Moov) -> anyhow::Result { - if self.config.passthrough { - // Build a single-track init segment (ftyp+moov) for this track. + // Build a single-track init segment (ftyp+moov) for this track. + { let ftyp = mp4_atom::Ftyp { major_brand: b"isom".into(), minor_version: 0x200, @@ -268,8 +224,6 @@ impl Fmp4 { let init_data = base64::engine::general_purpose::STANDARD.encode(&buf); Ok(Container::Cmaf { init_data }) - } else { - Ok(Container::Legacy) } } @@ -488,7 +442,7 @@ impl Fmp4 { Ok(config) } - // Extract all frames out of an mdat atom. + // Extract all frames out of an mdat atom using CMAF passthrough. fn extract(&mut self, mdat: Mdat, mdat_raw: &[u8]) -> anyhow::Result<()> { let moov = self.moov.as_ref().context("missing moov box")?; let moof = self.moof.take().context("missing moof box")?; @@ -521,13 +475,13 @@ impl Fmp4 { let timescale = trak.mdia.mdhd.timescale as u64; let mut offset = traf.tfhd.base_data_offset.unwrap_or_default() as usize; + let mut track_data_start: Option = None; if traf.trun.is_empty() { anyhow::bail!("missing trun box"); } // Keep track of the minimum and maximum timestamp for this track to compute the jitter. - // Ideally these should both be the same value (a single frame lul). let mut min_timestamp = None; let mut max_timestamp = None; let mut contains_keyframe = false; @@ -537,24 +491,24 @@ impl Fmp4 { if let Some(data_offset) = trun.data_offset { let base_offset = tfhd.base_data_offset.unwrap_or_default() as usize; - // This is relative to the start of the MOOF, not the MDAT. - // Note: The trun data offset can be negative, but... that's not supported here. let data_offset: usize = data_offset.try_into().context("invalid data offset")?; - // Use checked arithmetic to prevent underflow let relative_offset = data_offset .checked_sub(moof_size) .and_then(|v| v.checked_sub(header_size)) .context("invalid data offset: underflow")?; - // Reset offset if the TRUN has a data offset offset = base_offset .checked_add(relative_offset) .context("invalid data offset: overflow")?; } + // Capture the actual start offset for this traf before consuming samples + if track_data_start.is_none() { + track_data_start = Some(offset); + } + for entry in &trun.entries { - // Use the moof defaults if the sample doesn't have its own values. let flags = entry .flags .unwrap_or(tfhd.default_sample_flags.unwrap_or(default_sample_flags)); @@ -574,49 +528,15 @@ impl Fmp4 { let keyframe = match track.kind { TrackKind::Video => { - // https://chromium.googlesource.com/chromium/src/media/+/master/formats/mp4/track_run_iterator.cc#177 - let keyframe = (flags >> 24) & 0x3 == 0x2; // kSampleDependsOnNoOther - let non_sync = (flags >> 16) & 0x1 == 0x1; // kSampleIsNonSyncSample - + let keyframe = (flags >> 24) & 0x3 == 0x2; + let non_sync = (flags >> 16) & 0x1 == 0x1; keyframe && !non_sync } - TrackKind::Audio => { - // Audio frames are always keyframes. - // TODO: Optionally bundle audio frames into groups to - true - } + TrackKind::Audio => true, }; contains_keyframe |= keyframe; - if !self.config.passthrough { - // TODO Avoid a copy if mp4-atom switches to using Bytes? - let payload = Bytes::copy_from_slice(&mdat.data[offset..(offset + size)]); - - let frame = hang::container::Frame { - timestamp, - payload: payload.into(), - }; - - match &mut track.producer { - Fmp4Producer::Manual { track: raw, group } => { - let mut g = if keyframe { - if let Some(mut prev) = group.take() { - prev.finish()?; - } - raw.append_group()? - } else { - group.take().context("no keyframe at start")? - }; - frame.encode(&mut g)?; - *group = Some(g); - } - Fmp4Producer::Ordered(ordered) => { - ordered.write(frame)?; - } - } - } - if timestamp >= max_timestamp.unwrap_or(Timestamp::ZERO) { max_timestamp = Some(timestamp); } @@ -638,45 +558,86 @@ impl Fmp4 { } } - // If we're doing passthrough mode, then we write one giant fragment instead of individual frames. - if self.config.passthrough { - let Fmp4Producer::Manual { track: raw, group } = &mut track.producer else { - unreachable!("passthrough always uses Manual"); - }; - - let mut g = if contains_keyframe { - if let Some(mut prev) = group.take() { - prev.finish()?; - } - raw.append_group()? - } else { - group.take().context("no keyframe at start")? - }; + // Build a per-track moof containing only this traf, and a per-track mdat + // with only the samples belonging to this track. + let single_traf_moof = Moof { + mfhd: moof.mfhd.clone(), + traf: vec![traf.clone()], + }; - let moof_raw = self.moof_raw.as_ref().context("missing moof box")?; + // Compute the data range within the original mdat for this traf's samples. + let track_data_start = track_data_start.unwrap_or(0); + let track_data_end = offset; // offset was advanced past all samples above - // To avoid an extra allocation, we use the chunked API to write the moof and mdat atoms separately. - let mut frame = g.create_frame(moq_lite::Frame { - size: moof_raw.len() as u64 + mdat_raw.len() as u64, - })?; + // Adjust the trun data_offset to point into the new per-track mdat. + let mut adjusted_moof = single_traf_moof; + let mut moof_buf = Vec::new(); + adjusted_moof.encode(&mut moof_buf)?; + let new_moof_size = moof_buf.len(); - frame.write(moof_raw.clone())?; - frame.write(Bytes::copy_from_slice(mdat_raw))?; - frame.finish()?; + // Slice out this track's data from the full mdat + let track_mdat_data = if track_data_start < mdat.data.len() && track_data_end <= mdat.data.len() { + &mdat.data[track_data_start..track_data_end] + } else { + // Fallback: use the full mdat data (single-track case) + &mdat.data[..] + }; - *group = Some(g); + // Re-encode moof with corrected per-trun data_offset for the per-track fragment. + // Each trun's data_offset points to the start of that run's data within the new mdat. + let mdat_header_size_new = 8u64; // 4 bytes size + 4 bytes 'mdat' + let mut cumulative_offset = 0u64; + for traf_mut in &mut adjusted_moof.traf { + // Clear base_data_offset since data_offset is now relative to moof start + traf_mut.tfhd.base_data_offset = None; + + for trun_mut in &mut traf_mut.trun { + trun_mut.data_offset = + Some((new_moof_size as u64 + mdat_header_size_new + cumulative_offset) as i32); + + // Advance past this trun's sample data + let trun_data_size: u64 = trun_mut + .entries + .iter() + .map(|e| { + e.size + .unwrap_or(traf_mut.tfhd.default_sample_size.unwrap_or(default_sample_size)) as u64 + }) + .sum(); + cumulative_offset += trun_data_size; + } } + moof_buf.clear(); + adjusted_moof.encode(&mut moof_buf)?; + + let per_track_mdat = Mdat { + data: track_mdat_data.to_vec(), + }; + per_track_mdat.encode(&mut moof_buf)?; + + let fragment_bytes = Bytes::from(moof_buf); + + // Write the per-track fragment as a single MoQ frame (passthrough). + let mut g = if contains_keyframe { + if let Some(mut prev) = track.group.take() { + prev.finish()?; + } + track.track.append_group()? + } else { + track.group.take().context("no keyframe at start")? + }; + + g.write_frame(fragment_bytes)?; + + track.group = Some(g); + if let (Some(min), Some(max), Some(min_duration)) = (min_timestamp, max_timestamp, track.min_duration) { - // We report the minimum buffer required as the difference between the min and max frames. - // We also add the duration between frames to account for the frame rate. - // ex. for 2s fragments, this should be exactly 2s if we did everything correctly. let jitter = max - min + min_duration; if jitter < track.jitter.unwrap_or(Timestamp::MAX) { track.jitter = Some(jitter); - // Update the catalog with the new jitter let mut catalog = self.catalog.lock(); match track.kind { @@ -684,7 +645,7 @@ impl Fmp4 { let config = catalog .video .renditions - .get_mut(&track.producer.info().name) + .get_mut(&track.track.info.name) .context("missing video config")?; config.jitter = Some(jitter.convert()?); } @@ -692,7 +653,7 @@ impl Fmp4 { let config = catalog .audio .renditions - .get_mut(&track.producer.info().name) + .get_mut(&track.track.info.name) .context("missing audio config")?; config.jitter = Some(jitter.convert()?); } @@ -709,17 +670,10 @@ impl Fmp4 { /// Finish all tracks, flushing current groups. pub fn finish(&mut self) -> anyhow::Result<()> { for track in self.tracks.values_mut() { - match &mut track.producer { - Fmp4Producer::Manual { track: raw, group } => { - if let Some(mut g) = group.take() { - g.finish()?; - } - raw.finish()?; - } - Fmp4Producer::Ordered(ordered) => { - ordered.finish()?; - } + if let Some(mut g) = track.group.take() { + g.finish()?; } + track.track.finish()?; } Ok(()) } @@ -731,8 +685,8 @@ impl Drop for Fmp4 { for track in self.tracks.values() { match track.kind { - TrackKind::Video => catalog.video.remove_track(track.producer.info()).is_some(), - TrackKind::Audio => catalog.audio.remove_track(track.producer.info()).is_some(), + TrackKind::Video => catalog.video.remove_track(&track.track.info).is_some(), + TrackKind::Audio => catalog.audio.remove_track(&track.track.info).is_some(), }; } } diff --git a/rs/moq-mux/src/import/hev1.rs b/rs/moq-mux/src/producer/hev1.rs similarity index 100% rename from rs/moq-mux/src/import/hev1.rs rename to rs/moq-mux/src/producer/hev1.rs diff --git a/rs/moq-mux/src/import/hls.rs b/rs/moq-mux/src/producer/hls.rs similarity index 96% rename from rs/moq-mux/src/import/hls.rs rename to rs/moq-mux/src/producer/hls.rs index 99a009059..d262ba489 100644 --- a/rs/moq-mux/src/import/hls.rs +++ b/rs/moq-mux/src/producer/hls.rs @@ -19,7 +19,7 @@ use reqwest::Client; use tracing::{debug, info, warn}; use url::Url; -use super::{Fmp4, Fmp4Config}; +use super::Fmp4; /// Configuration for the single-rendition HLS ingest loop. #[derive(Clone)] @@ -30,20 +30,11 @@ pub struct HlsConfig { /// An optional HTTP client to use for fetching the playlist and segments. /// If not provided, a default client will be created. pub client: Option, - - /// Enable passthrough mode for CMAF fragment transport. - /// When enabled, complete fMP4 fragments (moof+mdat) are transported directly - /// instead of being decomposed into individual samples. - pub passthrough: bool, } impl HlsConfig { pub fn new(playlist: String) -> Self { - Self { - playlist, - client: None, - passthrough: false, - } + Self { playlist, client: None } } /// Parse the playlist string into a URL. @@ -97,7 +88,6 @@ pub struct Hls { video: Vec, /// Optional audio track shared across variants. audio: Option, - passthrough: bool, } #[derive(Debug, Clone, Copy)] @@ -136,12 +126,10 @@ impl Hls { .build() .unwrap() }); - let passthrough = cfg.passthrough; Ok(Self { broadcast, catalog, video_importers: Vec::new(), - passthrough, audio_importer: None, client, base_url, @@ -487,13 +475,7 @@ impl Hls { /// independent while still contributing to the same shared catalog. fn ensure_video_importer_for(&mut self, index: usize) -> &mut Fmp4 { while self.video_importers.len() <= index { - let importer = Fmp4::new( - self.broadcast.clone(), - self.catalog.clone(), - Fmp4Config { - passthrough: self.passthrough, - }, - ); + let importer = Fmp4::new(self.broadcast.clone(), self.catalog.clone()); self.video_importers.push(importer); } @@ -502,9 +484,8 @@ impl Hls { /// Create or retrieve the fMP4 importer for the audio rendition. fn ensure_audio_importer(&mut self) -> &mut Fmp4 { - let passthrough = self.passthrough; self.audio_importer - .get_or_insert_with(|| Fmp4::new(self.broadcast.clone(), self.catalog.clone(), Fmp4Config { passthrough })) + .get_or_insert_with(|| Fmp4::new(self.broadcast.clone(), self.catalog.clone())) } #[cfg(test)] diff --git a/rs/moq-mux/src/import/mod.rs b/rs/moq-mux/src/producer/mod.rs similarity index 80% rename from rs/moq-mux/src/import/mod.rs rename to rs/moq-mux/src/producer/mod.rs index cc1073cb9..1cb44b854 100644 --- a/rs/moq-mux/src/import/mod.rs +++ b/rs/moq-mux/src/producer/mod.rs @@ -11,8 +11,9 @@ //! - `hev1`: H.265 with inline SPS/PPS. //! - `hls`: HLS playlist. //! -//! The [Decoder] module provides a generic interface for importing a stream of media. -//! If you know the format in advance, use the specific decoder instead. +//! The [Framed] and [Stream] types provide generic interfaces for importing media. +//! [Framed] is for formats with known frame boundaries, [Stream] for unknown boundaries. +//! If you know the format in advance, use the specific codec module instead. mod aac; #[cfg(any(feature = "h264", feature = "h265"))] diff --git a/rs/moq-mux/src/import/opus.rs b/rs/moq-mux/src/producer/opus.rs similarity index 100% rename from rs/moq-mux/src/import/opus.rs rename to rs/moq-mux/src/producer/opus.rs diff --git a/rs/moq-mux/src/import/test/av1.mp4 b/rs/moq-mux/src/producer/test/av1.mp4 similarity index 100% rename from rs/moq-mux/src/import/test/av1.mp4 rename to rs/moq-mux/src/producer/test/av1.mp4 diff --git a/rs/moq-mux/src/import/test/bbb.mp4 b/rs/moq-mux/src/producer/test/bbb.mp4 similarity index 100% rename from rs/moq-mux/src/import/test/bbb.mp4 rename to rs/moq-mux/src/producer/test/bbb.mp4 diff --git a/rs/moq-mux/src/import/test/mod.rs b/rs/moq-mux/src/producer/test/mod.rs similarity index 78% rename from rs/moq-mux/src/import/test/mod.rs rename to rs/moq-mux/src/producer/test/mod.rs index ddac02624..033383517 100644 --- a/rs/moq-mux/src/import/test/mod.rs +++ b/rs/moq-mux/src/producer/test/mod.rs @@ -2,12 +2,11 @@ use base64::Engine; use hang::catalog::Container; use mp4_atom::{Decode, Encode}; -fn run_fmp4(data: &[u8], passthrough: bool) -> hang::Catalog { +fn run_fmp4(data: &[u8]) -> hang::Catalog { let mut broadcast = moq_lite::Broadcast::new().produce(); let catalog = crate::CatalogProducer::new(&mut broadcast).unwrap(); - let config = super::Fmp4Config { passthrough }; - let mut fmp4 = super::Fmp4::new(broadcast, catalog.clone(), config); + let mut fmp4 = super::Fmp4::new(broadcast, catalog.clone()); let mut buf = bytes::BytesMut::from(data); // Ignore errors from incomplete/malformed trailing fragments in test files. @@ -27,9 +26,9 @@ fn decode_init_data(init_data: &str) -> (mp4_atom::Ftyp, mp4_atom::Moov) { } #[test] -fn test_bbb_passthrough_catalog() { +fn test_bbb_catalog() { let data = include_bytes!("bbb.mp4"); - let catalog = run_fmp4(data, true); + let catalog = run_fmp4(data); assert_eq!(catalog.video.renditions.len(), 1); assert_eq!(catalog.audio.renditions.len(), 1); @@ -48,9 +47,9 @@ fn test_bbb_passthrough_catalog() { } #[test] -fn test_bbb_passthrough_init_data_roundtrip() { +fn test_bbb_init_data_roundtrip() { let data = include_bytes!("bbb.mp4"); - let catalog = run_fmp4(data, true); + let catalog = run_fmp4(data); // Check video init data let video = catalog.video.renditions.values().next().unwrap(); @@ -90,30 +89,9 @@ fn test_bbb_passthrough_init_data_roundtrip() { } #[test] -fn test_bbb_legacy_catalog() { - let data = include_bytes!("bbb.mp4"); - let catalog = run_fmp4(data, false); - - assert_eq!(catalog.video.renditions.len(), 1); - assert_eq!(catalog.audio.renditions.len(), 1); - - let video = catalog.video.renditions.values().next().unwrap(); - assert_eq!(video.codec.to_string(), "avc1.64001f"); - assert_eq!(video.coded_width, Some(1280)); - assert_eq!(video.coded_height, Some(720)); - assert!(matches!(video.container, Container::Legacy)); - - let audio = catalog.audio.renditions.values().next().unwrap(); - assert_eq!(audio.codec.to_string(), "mp4a.40.2"); - assert_eq!(audio.sample_rate, 44100); - assert_eq!(audio.channel_count, 2); - assert!(matches!(audio.container, Container::Legacy)); -} - -#[test] -fn test_av1_passthrough_catalog() { +fn test_av1_catalog() { let data = include_bytes!("av1.mp4"); - let catalog = run_fmp4(data, true); + let catalog = run_fmp4(data); assert_eq!(catalog.video.renditions.len(), 1); assert_eq!(catalog.audio.renditions.len(), 0); @@ -134,9 +112,9 @@ fn test_av1_passthrough_catalog() { } #[test] -fn test_vp9_passthrough_catalog() { +fn test_vp9_catalog() { let data = include_bytes!("vp9.mp4"); - let catalog = run_fmp4(data, true); + let catalog = run_fmp4(data); assert_eq!(catalog.video.renditions.len(), 1); assert_eq!(catalog.audio.renditions.len(), 0); diff --git a/rs/moq-mux/src/import/test/vp9.mp4 b/rs/moq-mux/src/producer/test/vp9.mp4 similarity index 100% rename from rs/moq-mux/src/import/test/vp9.mp4 rename to rs/moq-mux/src/producer/test/vp9.mp4 From 2626a255616689c0e8444557e676f87633a4e031 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 19 Mar 2026 14:27:08 -0700 Subject: [PATCH 08/25] Move sync to props and add dynamic SyncTrack API (#1138) Co-authored-by: Claude Opus 4.6 (1M context) --- js/watch/src/audio/source.ts | 14 +++++++--- js/watch/src/backend.ts | 9 ++++-- js/watch/src/mse.ts | 8 ++++-- js/watch/src/sync.ts | 54 ++++++++++++++++++++++++++++-------- js/watch/src/video/source.ts | 13 ++++++--- 5 files changed, 73 insertions(+), 25 deletions(-) diff --git a/js/watch/src/audio/source.ts b/js/watch/src/audio/source.ts index d112c44fa..6d6d9c915 100644 --- a/js/watch/src/audio/source.ts +++ b/js/watch/src/audio/source.ts @@ -2,7 +2,7 @@ import type * as Catalog from "@moq/hang/catalog"; import type * as Moq from "@moq/lite"; import { Effect, type Getter, Signal } from "@moq/signals"; import type { Broadcast } from "../broadcast"; -import type { Sync } from "../sync"; +import { Sync, type SyncTrack } from "../sync"; export type Target = { // Optional manual override for the selected rendition name. @@ -22,6 +22,9 @@ export type SourceProps = { // A function that checks if an audio configuration is supported by the backend. supported?: Supported; + + // Shared Sync instance for synchronizing playback across tracks. Defaults to a standalone Sync. + sync?: Sync; }; /** @@ -48,11 +51,13 @@ export class Source { // Used to target a latency and synchronize playback of video with audio. sync: Sync; + #syncTrack: SyncTrack; #signals = new Effect(); - constructor(sync: Sync, props?: SourceProps) { - this.sync = sync; + constructor(props: SourceProps) { + this.sync = props.sync ?? new Sync(); + this.#syncTrack = this.sync.track(); this.broadcast = Signal.from(props?.broadcast); this.target = Signal.from(props?.target); @@ -113,7 +118,7 @@ export class Source { effect.set(this.#track, selected.track); effect.set(this.#config, selected.config); - effect.set(this.sync.audio, selected.config.jitter as Moq.Time.Milli | undefined); + this.#syncTrack.set(selected.config.jitter as Moq.Time.Milli | undefined); } /** @@ -141,6 +146,7 @@ export class Source { } close(): void { + this.#syncTrack.close(); this.#signals.close(); } } diff --git a/js/watch/src/backend.ts b/js/watch/src/backend.ts index 1a6ebd272..27cacc5c0 100644 --- a/js/watch/src/backend.ts +++ b/js/watch/src/backend.ts @@ -119,10 +119,12 @@ export class MultiBackend implements Backend { this.jitter = Signal.from(props?.jitter ?? (100 as Moq.Time.Milli)); this.#sync = new Sync({ jitter: this.jitter }); - this.#videoSource = new Video.Source(this.#sync, { + this.#videoSource = new Video.Source({ + sync: this.#sync, broadcast: this.broadcast, }); - this.#audioSource = new Audio.Source(this.#sync, { + this.#audioSource = new Audio.Source({ + sync: this.#sync, broadcast: this.broadcast, }); @@ -175,7 +177,8 @@ export class MultiBackend implements Backend { } #runMse(effect: Effect, element: HTMLVideoElement): void { - const mse = new Muxer(this.#sync, { + const mse = new Muxer({ + sync: this.#sync, paused: this.paused, element, }); diff --git a/js/watch/src/mse.ts b/js/watch/src/mse.ts index 615fe13d8..632c31de8 100644 --- a/js/watch/src/mse.ts +++ b/js/watch/src/mse.ts @@ -1,10 +1,12 @@ import { Time } from "@moq/lite"; import { Effect, type Getter, Signal } from "@moq/signals"; -import type { Sync } from "./sync"; +import { Sync } from "./sync"; export type MuxerProps = { element?: HTMLMediaElement | Signal; paused?: boolean | Signal; + // Shared Sync instance for synchronizing playback across tracks. Defaults to a standalone Sync. + sync?: Sync; }; /** @@ -23,10 +25,10 @@ export class Muxer { #signals = new Effect(); - constructor(sync: Sync, props?: MuxerProps) { + constructor(props: MuxerProps) { this.element = Signal.from(props?.element); this.paused = Signal.from(props?.paused ?? false); - this.#sync = sync; + this.#sync = props.sync ?? new Sync(); this.#signals.run(this.#runMediaSource.bind(this)); this.#signals.run(this.#runSkip.bind(this)); diff --git a/js/watch/src/sync.ts b/js/watch/src/sync.ts index 567bc5d75..8233f8659 100644 --- a/js/watch/src/sync.ts +++ b/js/watch/src/sync.ts @@ -1,10 +1,29 @@ import { Time } from "@moq/lite"; -import { Effect, Signal } from "@moq/signals"; +import { Effect, type Getter, Signal } from "@moq/signals"; + +export class SyncTrack { + #jitter = new Signal(undefined); + readonly jitter: Getter = this.#jitter; + #onClose: () => void; + #closed = false; + + constructor(onClose: () => void) { + this.#onClose = onClose; + } + + set(jitter: Time.Milli | undefined): void { + this.#jitter.set(jitter); + } + + close(): void { + if (this.#closed) return; + this.#closed = true; + this.#onClose(); + } +} export interface SyncProps { jitter?: Time.Milli | Signal; - audio?: Time.Milli | Signal; - video?: Time.Milli | Signal; } export class Sync { @@ -17,9 +36,9 @@ export class Sync { // The minimum buffer size, to account for network jitter. jitter: Signal; - // Any additional delay required for audio or video. - audio: Signal; - video: Signal; + // Dynamic set of track consumers. + #tracks = new Set(); + #tracksVersion = new Signal(0); // The buffer required, based on both audio and video. #latency = new Signal(Time.Milli.zero); @@ -33,20 +52,33 @@ export class Sync { constructor(props?: SyncProps) { this.jitter = Signal.from(props?.jitter ?? (100 as Time.Milli)); - this.audio = Signal.from(props?.audio); - this.video = Signal.from(props?.video); this.#update = Promise.withResolvers(); this.signals.run(this.#runLatency.bind(this)); } + track(): SyncTrack { + const t = new SyncTrack(() => { + this.#tracks.delete(t); + this.#tracksVersion.update((v) => v + 1); + }); + this.#tracks.add(t); + this.#tracksVersion.update((v) => v + 1); + return t; + } + #runLatency(effect: Effect): void { const jitter = effect.get(this.jitter); - const video = effect.get(this.video) ?? Time.Milli.zero; - const audio = effect.get(this.audio) ?? Time.Milli.zero; + effect.get(this.#tracksVersion); + + let max = Time.Milli.zero; + for (const t of this.#tracks) { + const v = effect.get(t.jitter) ?? Time.Milli.zero; + max = Time.Milli.max(max, v); + } - const latency = Time.Milli.add(Time.Milli.max(video, audio), jitter); + const latency = Time.Milli.add(max, jitter); this.#latency.set(latency); this.#update.resolve(); diff --git a/js/watch/src/video/source.ts b/js/watch/src/video/source.ts index e3c0c9b57..26cf6bdd4 100644 --- a/js/watch/src/video/source.ts +++ b/js/watch/src/video/source.ts @@ -2,7 +2,7 @@ import type * as Catalog from "@moq/hang/catalog"; import type * as Moq from "@moq/lite"; import { Effect, type Getter, Signal } from "@moq/signals"; import type { Broadcast } from "../broadcast"; -import type { Sync } from "../sync"; +import { Sync, type SyncTrack } from "../sync"; /** * A function that checks if a video configuration is supported by the backend. @@ -13,6 +13,8 @@ export type SourceProps = { broadcast?: Broadcast | Signal; target?: Target | Signal; supported?: Supported; + // Shared Sync instance for synchronizing playback across tracks. Defaults to a standalone Sync. + sync?: Sync; }; export type Target = { @@ -159,14 +161,16 @@ export class Source { readonly config: Getter = this.#config; sync: Sync; + #syncTrack: SyncTrack; supported: Signal; #signals = new Effect(); - constructor(sync: Sync, props?: SourceProps) { + constructor(props: SourceProps) { this.broadcast = Signal.from(props?.broadcast); this.target = Signal.from(props?.target); - this.sync = sync; + this.sync = props.sync ?? new Sync(); + this.#syncTrack = this.sync.track(); this.supported = Signal.from(props?.supported); this.#signals.run(this.#runCatalog.bind(this)); @@ -221,7 +225,7 @@ export class Source { effect.set(this.#track, selected); effect.set(this.#config, config); - effect.set(this.sync.video, config.jitter as Moq.Time.Milli | undefined); + this.#syncTrack.set(config.jitter as Moq.Time.Milli | undefined); } /** @@ -268,6 +272,7 @@ export class Source { } close(): void { + this.#syncTrack.close(); this.#signals.close(); } } From 997e161e973337537fa9148b02618ef88050ac11 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 22 Mar 2026 14:41:51 -0700 Subject: [PATCH 09/25] Replace monolithic catalog with generic section-based registry The hang catalog previously hardcoded application-specific sections (chat, user, preview, location, capabilities). This replaces the monolithic Catalog/Root type with a generic Section registry where sections are identified by name + schema pairs, enabling custom applications to define their own sections. Key changes: - Add Section type pairing a name with a typed schema (Rust: serde, JS: zod) - Add CatalogWriter for producing catalogs with typed sections - Add CatalogReader with per-section change notifications (conducer/signals) - Predefined VIDEO/AUDIO sections (not auto-registered) - Remove app-specific sections from @moq/hang (chat, user, preview, etc.) - Migrate moq-mux, moq-cli, libmoq, moq-ffi to section-based API - Migrate js/publish and js/watch to use CatalogWriter/CatalogReader Co-Authored-By: Claude Opus 4.6 (1M context) --- Cargo.lock | 5 + bun.lock | 3 +- js/hang/src/catalog/audio.ts | 4 + js/hang/src/catalog/capabilities.ts | 22 --- js/hang/src/catalog/chat.ts | 9 - js/hang/src/catalog/index.ts | 9 +- js/hang/src/catalog/location.ts | 45 ----- js/hang/src/catalog/preview.ts | 15 -- js/hang/src/catalog/reader.ts | 67 ++++++++ js/hang/src/catalog/root.ts | 45 ----- js/hang/src/catalog/section.ts | 15 ++ js/hang/src/catalog/user.ts | 10 -- js/hang/src/catalog/video.ts | 4 + js/hang/src/catalog/writer.ts | 44 +++++ js/publish/src/audio/encoder.ts | 26 +-- js/publish/src/broadcast.ts | 38 +++-- js/publish/src/chat/index.ts | 11 +- js/publish/src/chat/message.ts | 6 +- js/publish/src/chat/typing.ts | 6 +- js/publish/src/location/index.ts | 6 +- js/publish/src/location/peers.ts | 14 +- js/publish/src/location/types.ts | 8 + js/publish/src/location/window.ts | 21 ++- js/publish/src/preview.ts | 20 ++- js/publish/src/user.ts | 10 +- js/publish/src/video/encoder.ts | 20 +-- js/publish/src/video/index.ts | 14 +- js/watch/package.json | 3 +- js/watch/src/audio/source.ts | 2 +- js/watch/src/broadcast.ts | 58 ++++--- js/watch/src/chat/index.ts | 12 +- js/watch/src/chat/message.ts | 15 +- js/watch/src/chat/typing.ts | 15 +- js/watch/src/location/index.ts | 10 +- js/watch/src/location/peers.ts | 19 ++- js/watch/src/location/window.ts | 23 +-- js/watch/src/preview.ts | 25 ++- js/watch/src/sections.ts | 63 +++++++ js/watch/src/user.ts | 13 +- js/watch/src/video/source.ts | 2 +- rs/hang/Cargo.toml | 1 + rs/hang/examples/subscribe.rs | 18 +- rs/hang/examples/video.rs | 24 +-- rs/hang/src/catalog/audio/mod.rs | 5 +- rs/hang/src/catalog/chat.rs | 10 -- rs/hang/src/catalog/consumer.rs | 60 +++++-- rs/hang/src/catalog/mod.rs | 18 +- rs/hang/src/catalog/preview.rs | 17 -- rs/hang/src/catalog/reader.rs | 150 +++++++++++++++++ rs/hang/src/catalog/root.rs | 183 --------------------- rs/hang/src/catalog/section.rs | 19 +++ rs/hang/src/catalog/user.rs | 12 -- rs/hang/src/catalog/video/mod.rs | 5 +- rs/hang/src/catalog/writer.rs | 74 +++++++++ rs/hang/src/error.rs | 4 + rs/hang/src/lib.rs | 2 +- rs/libmoq/Cargo.toml | 1 + rs/libmoq/src/consume.rs | 132 ++++++++------- rs/moq-cli/src/subscribe.rs | 25 ++- rs/moq-ffi/Cargo.toml | 1 + rs/moq-ffi/src/consumer.rs | 47 +++++- rs/moq-ffi/src/media.rs | 18 +- rs/moq-mux/Cargo.toml | 2 + rs/moq-mux/src/catalog.rs | 126 ++++++-------- rs/moq-mux/src/consumer/fmp4.rs | 16 +- rs/moq-mux/src/convert/fmp4.rs | 236 +++++++++++++++------------ rs/moq-mux/src/convert/hang.rs | 244 ++++++++++++++++------------ rs/moq-mux/src/convert/test.rs | 17 +- rs/moq-mux/src/msf.rs | 77 ++++----- rs/moq-mux/src/producer/aac.rs | 53 +++--- rs/moq-mux/src/producer/av01.rs | 43 +++-- rs/moq-mux/src/producer/avc3.rs | 27 ++- rs/moq-mux/src/producer/fmp4.rs | 44 +++-- rs/moq-mux/src/producer/hev1.rs | 27 ++- rs/moq-mux/src/producer/opus.rs | 49 ++++-- rs/moq-mux/src/producer/test/mod.rs | 24 ++- 76 files changed, 1475 insertions(+), 1093 deletions(-) delete mode 100644 js/hang/src/catalog/capabilities.ts delete mode 100644 js/hang/src/catalog/chat.ts delete mode 100644 js/hang/src/catalog/location.ts delete mode 100644 js/hang/src/catalog/preview.ts create mode 100644 js/hang/src/catalog/reader.ts delete mode 100644 js/hang/src/catalog/root.ts create mode 100644 js/hang/src/catalog/section.ts delete mode 100644 js/hang/src/catalog/user.ts create mode 100644 js/hang/src/catalog/writer.ts create mode 100644 js/publish/src/location/types.ts create mode 100644 js/watch/src/sections.ts delete mode 100644 rs/hang/src/catalog/chat.rs delete mode 100644 rs/hang/src/catalog/preview.rs create mode 100644 rs/hang/src/catalog/reader.rs delete mode 100644 rs/hang/src/catalog/root.rs create mode 100644 rs/hang/src/catalog/section.rs delete mode 100644 rs/hang/src/catalog/user.rs create mode 100644 rs/hang/src/catalog/writer.rs diff --git a/Cargo.lock b/Cargo.lock index 94835ee13..badcec3da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2109,6 +2109,7 @@ dependencies = [ "anyhow", "buf-list", "bytes", + "conducer", "derive_more", "hex", "lazy_static", @@ -3046,6 +3047,7 @@ dependencies = [ "moq-lite", "moq-mux", "moq-native", + "serde_json", "thiserror 2.0.18", "tokio", "tracing", @@ -3287,6 +3289,7 @@ dependencies = [ "moq-mux", "moq-native", "pollster", + "serde_json", "thiserror 2.0.18", "tokio", "tracing", @@ -3340,6 +3343,8 @@ dependencies = [ "reqwest 0.12.28", "scuffle-av1", "scuffle-h265", + "serde", + "serde_json", "thiserror 2.0.18", "tokio", "tracing", diff --git a/bun.lock b/bun.lock index ac7c11629..375d3c943 100644 --- a/bun.lock +++ b/bun.lock @@ -77,7 +77,7 @@ }, "js/lite": { "name": "@moq/lite", - "version": "0.1.6", + "version": "0.1.7", "dependencies": { "@moq/qmux": "^0.0.6", "@moq/signals": "workspace:*", @@ -175,6 +175,7 @@ "@moq/lite": "workspace:^", "@moq/signals": "workspace:^", "@moq/ui-core": "workspace:^", + "zod": "^4.1.5", }, "devDependencies": { "@types/audioworklet": "^0.0.77", diff --git a/js/hang/src/catalog/audio.ts b/js/hang/src/catalog/audio.ts index 91ed39167..3c313ef08 100644 --- a/js/hang/src/catalog/audio.ts +++ b/js/hang/src/catalog/audio.ts @@ -1,6 +1,7 @@ import { z } from "zod"; import { ContainerSchema } from "./container"; import { u53Schema } from "./integers"; +import { Section } from "./section"; // Backwards compatibility: old track schema const TrackSchema = z.object({ @@ -60,3 +61,6 @@ export const AudioSchema = z export type Audio = z.infer; export type AudioConfig = z.infer; + +/// Predefined section for audio catalog data. +export const AUDIO = new Section("audio", AudioSchema); diff --git a/js/hang/src/catalog/capabilities.ts b/js/hang/src/catalog/capabilities.ts deleted file mode 100644 index 785006ef7..000000000 --- a/js/hang/src/catalog/capabilities.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { z } from "zod"; - -export const VideoCapabilitiesSchema = z.object({ - hardware: z.array(z.string()).optional(), - software: z.array(z.string()).optional(), - unsupported: z.array(z.string()).optional(), -}); - -export const AudioCapabilitiesSchema = z.object({ - hardware: z.array(z.string()).optional(), - software: z.array(z.string()).optional(), - unsupported: z.array(z.string()).optional(), -}); - -export const CapabilitiesSchema = z.object({ - video: VideoCapabilitiesSchema.optional(), - audio: AudioCapabilitiesSchema.optional(), -}); - -export type Capabilities = z.infer; -export type VideoCapabilities = z.infer; -export type AudioCapabilities = z.infer; diff --git a/js/hang/src/catalog/chat.ts b/js/hang/src/catalog/chat.ts deleted file mode 100644 index 9950fa3eb..000000000 --- a/js/hang/src/catalog/chat.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { z } from "zod"; -import { TrackSchema } from "./track"; - -export const ChatSchema = z.object({ - message: TrackSchema.optional(), - typing: TrackSchema.optional(), -}); - -export type Chat = z.infer; diff --git a/js/hang/src/catalog/index.ts b/js/hang/src/catalog/index.ts index ecf4e9daf..86aee07be 100644 --- a/js/hang/src/catalog/index.ts +++ b/js/hang/src/catalog/index.ts @@ -1,12 +1,9 @@ export * from "./audio"; -export * from "./capabilities"; -export * from "./chat"; export * from "./container"; export * from "./integers"; -export * from "./location"; -export * from "./preview"; export * from "./priority"; -export * from "./root"; +export * from "./reader"; +export * from "./section"; export * from "./track"; -export * from "./user"; export * from "./video"; +export * from "./writer"; diff --git a/js/hang/src/catalog/location.ts b/js/hang/src/catalog/location.ts deleted file mode 100644 index e7eb014fd..000000000 --- a/js/hang/src/catalog/location.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { z } from "zod"; -import { TrackSchema } from "./track"; - -export const PositionSchema = z.object({ - // The relative X position of the broadcast, from -1 to +1. - // This should be used for audio panning but can also be used for video positioning. - x: z.number().optional(), - - // The relative Y position of the broadcast, from -1 to +1. - // This can be used for video positioning, and maybe audio panning. - y: z.number().optional(), - - // The relative Z index of the broadcast, where larger values are closer to the viewer. - // This is used to break ties when there are multiple broadcasts at the same position. - z: z.number().optional(), - - // The scale of the broadcast, where 1 is 100% - s: z.number().optional(), -}); - -export const LocationSchema = z.object({ - // The initial position of the broadcaster, from -1 to +1 in both dimensions. - // If not provided, then the broadcaster is assumed to be at (0,0) - // This should be used for audio panning but can also be used for video positioning. - initial: PositionSchema.optional(), - - // If provided, then updates to the position are done via a separate Moq track. - // This is used to avoid a full catalog update every time we want to update a few bytes. - // TODO: These updates currently use JSON for simplicity, but we should use a binary format. - track: TrackSchema.optional(), - - // If set, then this broadcaster allows other peers to request position updates via this handle. - // We will have to discover and subscribe to their position updates. - handle: z.string().optional(), - - // If provided, this broadcaster is signaling the location of other peers. - // The payload is a JSON blob keyed by handle for each peer. - peers: TrackSchema.optional(), -}); - -export type Location = z.infer; -export type Position = z.infer; - -export const PeersSchema = z.record(z.string(), PositionSchema); -export type Peers = z.infer; diff --git a/js/hang/src/catalog/preview.ts b/js/hang/src/catalog/preview.ts deleted file mode 100644 index 75e220715..000000000 --- a/js/hang/src/catalog/preview.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { z } from "zod"; - -export const PreviewSchema = z.object({ - name: z.string().optional(), // name - avatar: z.string().optional(), // avatar - - audio: z.boolean().optional(), // audio enabled - video: z.boolean().optional(), // video enabled - - typing: z.boolean().optional(), // actively typing - chat: z.boolean().optional(), // chatted recently - screen: z.boolean().optional(), // screen sharing -}); - -export type Preview = z.infer; diff --git a/js/hang/src/catalog/reader.ts b/js/hang/src/catalog/reader.ts new file mode 100644 index 000000000..3987757e5 --- /dev/null +++ b/js/hang/src/catalog/reader.ts @@ -0,0 +1,67 @@ +import type * as Moq from "@moq/lite"; +import { type Effect, type Getter, Signal } from "@moq/signals"; +import type { z } from "zod"; +import type { Section } from "./section"; + +/// A catalog reader that provides per-section change notifications. +/// +/// Sections are registered with a name and Zod schema. When the catalog track +/// receives a new frame, JSON is parsed and each registered section is validated +/// and updated. Signal equality checking ensures subscribers only fire when +/// their specific section's value actually changed. +export class CatalogReader { + // biome-ignore lint/suspicious/noExplicitAny: we store heterogeneous section types + #sections = new Map; signal: Signal }>(); + + /// Register interest in a section. Returns a Getter. + /// + /// The getter updates when the catalog is re-fetched and this section's value differs. + section(def: Section): Getter { + const existing = this.#sections.get(def.name); + if (existing) return existing.signal as Getter; + + const signal = new Signal(undefined); + this.#sections.set(def.name, { schema: def.schema, signal }); + return signal; + } + + /// Start consuming from a MoQ track. + /// + /// Spawns an async loop that reads frames, parses JSON, and updates + /// per-section signals. Unregistered keys in the JSON are ignored. + consume(track: Moq.Track, effect: Effect): void { + effect.spawn(async () => { + try { + for (;;) { + const frame = await Promise.race([effect.cancel, track.readFrame()]); + if (!frame) break; + + const decoder = new TextDecoder(); + const str = decoder.decode(frame); + const json = JSON.parse(str); + + for (const [name, { schema, signal }] of this.#sections) { + const raw = json[name]; + if (raw !== undefined) { + try { + const parsed = schema.parse(raw); + signal.set(parsed); + } catch (err) { + console.warn(`invalid catalog section "${name}"`, err); + } + } else { + signal.set(undefined); + } + } + } + } catch (err) { + console.warn("error reading catalog", err); + } finally { + // Clear all sections when the track ends + for (const { signal } of this.#sections.values()) { + signal.set(undefined); + } + } + }); + } +} diff --git a/js/hang/src/catalog/root.ts b/js/hang/src/catalog/root.ts deleted file mode 100644 index 7c9684c3e..000000000 --- a/js/hang/src/catalog/root.ts +++ /dev/null @@ -1,45 +0,0 @@ -import type * as Moq from "@moq/lite"; -import { z } from "zod"; - -import { AudioSchema } from "./audio"; -import { CapabilitiesSchema } from "./capabilities"; -import { ChatSchema } from "./chat"; -import { LocationSchema } from "./location"; -import { TrackSchema } from "./track"; -import { UserSchema } from "./user"; -import { VideoSchema } from "./video"; - -export const RootSchema = z.object({ - video: VideoSchema.optional(), - audio: AudioSchema.optional(), - location: LocationSchema.optional(), - user: UserSchema.optional(), - chat: ChatSchema.optional(), - capabilities: CapabilitiesSchema.optional(), - preview: TrackSchema.optional(), -}); - -export type Root = z.infer; - -export function encode(root: Root): Uint8Array { - const encoder = new TextEncoder(); - return encoder.encode(JSON.stringify(root)); -} - -export function decode(raw: Uint8Array): Root { - const decoder = new TextDecoder(); - const str = decoder.decode(raw); - try { - const json = JSON.parse(str); - return RootSchema.parse(json); - } catch (error) { - console.warn("invalid catalog", str); - throw error; - } -} - -export async function fetch(track: Moq.Track): Promise { - const frame = await track.readFrame(); - if (!frame) return undefined; - return decode(frame); -} diff --git a/js/hang/src/catalog/section.ts b/js/hang/src/catalog/section.ts new file mode 100644 index 000000000..72fa9bd2f --- /dev/null +++ b/js/hang/src/catalog/section.ts @@ -0,0 +1,15 @@ +import type { z } from "zod"; + +/// A section definition that pairs a JSON key name with a Zod schema. +/// +/// Used to register interest in specific catalog sections for reading or writing. +/// Audio and video sections are predefined but not registered by default. +export class Section { + readonly name: string; + readonly schema: z.ZodType; + + constructor(name: string, schema: z.ZodType) { + this.name = name; + this.schema = schema; + } +} diff --git a/js/hang/src/catalog/user.ts b/js/hang/src/catalog/user.ts deleted file mode 100644 index eab22b099..000000000 --- a/js/hang/src/catalog/user.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { z } from "zod"; - -export const UserSchema = z.object({ - id: z.string().optional(), - name: z.string().optional(), - avatar: z.string().optional(), // TODO allow using a track instead of a URL? - color: z.string().optional(), -}); - -export type User = z.infer; diff --git a/js/hang/src/catalog/video.ts b/js/hang/src/catalog/video.ts index f2db22edf..d68742fdc 100644 --- a/js/hang/src/catalog/video.ts +++ b/js/hang/src/catalog/video.ts @@ -1,6 +1,7 @@ import { z } from "zod"; import { ContainerSchema } from "./container"; import { u53Schema } from "./integers"; +import { Section } from "./section"; // Backwards compatibility: old track schema const TrackSchema = z.object({ @@ -103,3 +104,6 @@ export const VideoSchema = z export type Video = z.infer; export type VideoConfig = z.infer; + +/// Predefined section for video catalog data. +export const VIDEO = new Section("video", VideoSchema); diff --git a/js/hang/src/catalog/writer.ts b/js/hang/src/catalog/writer.ts new file mode 100644 index 000000000..b6863bc54 --- /dev/null +++ b/js/hang/src/catalog/writer.ts @@ -0,0 +1,44 @@ +import type * as Moq from "@moq/lite"; +import { type Effect, Signal } from "@moq/signals"; +import type { Section } from "./section"; + +/// A catalog writer that manages typed sections and serializes them to a MoQ track. +/// +/// Each section is a Signal that can be set independently. When served on a track, +/// changes are reactively detected and the full catalog JSON is re-serialized. +/// Microtask coalescing in Signal means multiple set() calls in the same tick +/// produce a single write. +export class CatalogWriter { + // biome-ignore lint/suspicious/noExplicitAny: we store heterogeneous section types + #sections = new Map }>(); + + /// Register a section for writing. Returns a Signal for read+write. + section(def: Section): Signal { + const existing = this.#sections.get(def.name); + if (existing) return existing.signal as Signal; + + const signal = new Signal(undefined); + this.#sections.set(def.name, { signal }); + return signal; + } + + /// Serve the catalog on a MoQ track. + /// + /// Uses Effect to reactively subscribe to all registered section signals. + /// When any signal changes, re-serializes and writes a new frame. + serve(track: Moq.Track, effect: Effect): void { + effect.run((inner) => { + const obj: Record = {}; + + for (const [name, { signal }] of this.#sections) { + const value = inner.get(signal); + if (value !== undefined) { + obj[name] = value; + } + } + + const encoder = new TextEncoder(); + track.writeFrame(encoder.encode(JSON.stringify(obj))); + }); + } +} diff --git a/js/publish/src/audio/encoder.ts b/js/publish/src/audio/encoder.ts index 3266d2bd9..8ba0f256a 100644 --- a/js/publish/src/audio/encoder.ts +++ b/js/publish/src/audio/encoder.ts @@ -1,5 +1,5 @@ -import * as Catalog from "@moq/hang/catalog"; -import * as Container from "@moq/hang/container"; +import { type Audio, type AudioConfig, type Container, PRIORITY, u53 } from "@moq/hang/catalog"; +import * as ContainerMod from "@moq/hang/container"; import * as Util from "@moq/hang/util"; import type * as Moq from "@moq/lite"; import { Time } from "@moq/lite"; @@ -25,12 +25,12 @@ export type EncoderProps = { // NOTE: Each frame is always flushed to the network immediately. groupDuration?: Time.Milli; - container?: Catalog.Container; + container?: Container; }; export class Encoder { static readonly TRACK = "audio/data"; - static readonly PRIORITY = Catalog.PRIORITY.audio; + static readonly PRIORITY = PRIORITY.audio; enabled: Signal; @@ -40,11 +40,11 @@ export class Encoder { source: Signal; - #catalog = new Signal(undefined); - readonly catalog: Getter = this.#catalog; + #catalog = new Signal