From c26d2d7855fe2eeafd49b5a597eea7d35b78edc8 Mon Sep 17 00:00:00 2001 From: Przemyslaw Gorszkowski Date: Fri, 12 Jun 2026 05:37:11 -0700 Subject: [PATCH] [GStreamer] Video element with canvas captureStream source should not use holepunch rendering https://bugs.webkit.org/show_bug.cgi?id=316297 Reviewed by Philippe Normand. Introduce a dedicated Canvas device type in CaptureDevice::DeviceType to distinguish CanvasCaptureMediaStreamTrack sources from Camera sources. Update all switch statements and type checks across the mediastream stack to handle Canvas as a video type. Disable GStreamer hole-punch rendering for canvas-captured streams since they render via a different path. No new tests. There is: ManualTests/mediastream/mediastream-canvas-to-video.html, which can be used to test this change. It should behave in the same way with: WEBKIT_GST_HOLE_PUNCH_QUIRK=fake and without it. * Source/WebCore/Modules/mediastream/CanvasCaptureMediaStreamTrack.cpp: (WebCore::CanvasCaptureMediaStreamTrack::Source::Source): * Source/WebCore/Modules/mediastream/CanvasCaptureMediaStreamTrack.h: * Source/WebCore/Modules/mediastream/MediaDeviceInfo.cpp: (WebCore::toMediaDeviceInfoKind): * Source/WebCore/Modules/mediastream/MediaStreamTrack.cpp: (WebCore::MediaStreamTrack::captureState): * Source/WebCore/dom/Document.cpp: (WebCore::updateCaptureSourceToPageMutedState): * Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.cpp: (WebCore::MediaPlayerPrivateGStreamer::isHolePunchRenderingEnabled const): * Source/WebCore/platform/mediastream/CaptureDevice.h: * Source/WebCore/platform/mediastream/RealtimeMediaSource.cpp: (WebCore::toSourceType): * Source/WebCore/platform/mediastream/gstreamer/GStreamerCaptureDeviceManager.cpp: (WebCore::GStreamerCaptureDeviceManager::refreshCaptureDevices): * Source/WebCore/platform/mock/MockRealtimeMediaSourceCenter.cpp: Canonical link: https://commits.webkit.org/315089@main --- .../mediastream/CanvasCaptureMediaStreamTrack.cpp | 2 +- .../mediastream/CanvasCaptureMediaStreamTrack.h | 1 + Source/WebCore/Modules/mediastream/MediaDeviceInfo.cpp | 1 + .../WebCore/Modules/mediastream/MediaStreamTrack.cpp | 8 ++++++++ Source/WebCore/dom/Document.cpp | 1 + .../graphics/gstreamer/MediaPlayerPrivateGStreamer.cpp | 10 ++++++++++ Source/WebCore/platform/mediastream/CaptureDevice.h | 2 +- .../platform/mediastream/RealtimeMediaSource.cpp | 1 + .../mediastream/cocoa/DisplayCaptureSourceCocoa.cpp | 1 + .../mediastream/gstreamer/GStreamerMockDevice.cpp | 1 + .../mediastream/mac/DisplayCaptureManagerCocoa.cpp | 1 + .../platform/mock/MockRealtimeMediaSourceCenter.cpp | 1 + Source/WebKit/GPUProcess/GPUConnectionToWebProcess.cpp | 1 + .../UIProcess/Cocoa/UserMediaCaptureManagerProxy.cpp | 1 + 14 files changed, 30 insertions(+), 2 deletions(-) diff --git a/Source/WebCore/Modules/mediastream/CanvasCaptureMediaStreamTrack.cpp b/Source/WebCore/Modules/mediastream/CanvasCaptureMediaStreamTrack.cpp index cdb2c6de1af05..509e6af20f1c5 100644 --- a/Source/WebCore/Modules/mediastream/CanvasCaptureMediaStreamTrack.cpp +++ b/Source/WebCore/Modules/mediastream/CanvasCaptureMediaStreamTrack.cpp @@ -76,7 +76,7 @@ Ref CanvasCaptureMediaStreamTrack::Source // FIXME: Give source id and name CanvasCaptureMediaStreamTrack::Source::Source(HTMLCanvasElement& canvas, std::optional&& frameRequestRate) - : RealtimeMediaSource(CaptureDevice { { }, CaptureDevice::DeviceType::Camera, "CanvasCaptureMediaStreamTrack"_s }) + : RealtimeMediaSource(CaptureDevice { { }, CaptureDevice::DeviceType::Canvas, "CanvasCaptureMediaStreamTrack"_s }) , m_frameRequestRate(WTFMove(frameRequestRate)) , m_requestFrameTimer(*this, &Source::requestFrameTimerFired) , m_captureCanvasTimer(*this, &Source::captureCanvas) diff --git a/Source/WebCore/Modules/mediastream/CanvasCaptureMediaStreamTrack.h b/Source/WebCore/Modules/mediastream/CanvasCaptureMediaStreamTrack.h index 82a412e148760..06c44d68b5a28 100644 --- a/Source/WebCore/Modules/mediastream/CanvasCaptureMediaStreamTrack.h +++ b/Source/WebCore/Modules/mediastream/CanvasCaptureMediaStreamTrack.h @@ -85,6 +85,7 @@ class CanvasCaptureMediaStreamTrack final : public MediaStreamTrack { void scheduleCaptureCanvas(); void captureCanvas(); void requestFrameTimerFired(); + CaptureDevice::DeviceType deviceType() const final { return CaptureDevice::DeviceType::Canvas; } bool m_shouldEmitFrame { true }; std::optional m_frameRequestRate; diff --git a/Source/WebCore/Modules/mediastream/MediaDeviceInfo.cpp b/Source/WebCore/Modules/mediastream/MediaDeviceInfo.cpp index 7b76bc8ae1267..7fc4e0d9a43aa 100644 --- a/Source/WebCore/Modules/mediastream/MediaDeviceInfo.cpp +++ b/Source/WebCore/Modules/mediastream/MediaDeviceInfo.cpp @@ -55,6 +55,7 @@ MediaDeviceInfo::Kind toMediaDeviceInfoKind(CaptureDevice::DeviceType type) case CaptureDevice::DeviceType::Speaker: return MediaDeviceInfo::Kind::Audiooutput; case CaptureDevice::DeviceType::Camera: + case CaptureDevice::DeviceType::Canvas: case CaptureDevice::DeviceType::Screen: case CaptureDevice::DeviceType::Window: return MediaDeviceInfo::Kind::Videoinput; diff --git a/Source/WebCore/Modules/mediastream/MediaStreamTrack.cpp b/Source/WebCore/Modules/mediastream/MediaStreamTrack.cpp index b28b18fa72ab4..516a84838ce3d 100644 --- a/Source/WebCore/Modules/mediastream/MediaStreamTrack.cpp +++ b/Source/WebCore/Modules/mediastream/MediaStreamTrack.cpp @@ -434,6 +434,14 @@ MediaProducerMediaStateFlags MediaStreamTrack::captureState(const RealtimeMediaS if (source.isProducingData()) return MediaProducerMediaState::HasActiveVideoCaptureDevice; break; + case CaptureDevice::DeviceType::Canvas: + if (source.muted()) + return MediaProducerMediaState::HasMutedVideoCaptureDevice; + if (source.interrupted()) + return MediaProducerMediaState::HasInterruptedVideoCaptureDevice; + if (source.isProducingData()) + return MediaProducerMediaState::HasActiveVideoCaptureDevice; + break; case CaptureDevice::DeviceType::Screen: if (source.muted()) return MediaProducerMediaState::HasMutedScreenCaptureDevice; diff --git a/Source/WebCore/dom/Document.cpp b/Source/WebCore/dom/Document.cpp index 7016739ac9a02..6ea3afa114a25 100644 --- a/Source/WebCore/dom/Document.cpp +++ b/Source/WebCore/dom/Document.cpp @@ -5419,6 +5419,7 @@ static void updateCaptureSourceToPageMutedState(Document& document, Page& page, source.setMuted(page.mutedState().contains(MediaProducerMutedState::AudioCaptureIsMuted) || (document.hidden() && document.settings().interruptAudioOnPageVisibilityChangeEnabled())); break; case CaptureDevice::DeviceType::Camera: + case CaptureDevice::DeviceType::Canvas: source.setMuted(page.mutedState().contains(MediaProducerMutedState::VideoCaptureIsMuted) || (document.hidden() && document.settings().interruptVideoOnPageVisibilityChangeEnabled())); break; case CaptureDevice::DeviceType::Screen: diff --git a/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.cpp b/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.cpp index 5227d5f372ae4..aef56943a2ecf 100644 --- a/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.cpp +++ b/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.cpp @@ -4501,6 +4501,16 @@ bool MediaPlayerPrivateGStreamer::isHolePunchRenderingEnabled() const if (m_quirksManagerForTesting) return m_quirksManagerForTesting->supportsVideoHolePunchRendering(); +#if ENABLE(MEDIA_STREAM) + if (m_streamPrivate) { + auto* videoTrack = m_streamPrivate->activeVideoTrack(); + if (!videoTrack) + return false; + + return videoTrack->deviceType() != CaptureDevice::DeviceType::Canvas && GStreamerQuirksManager::singleton().supportsVideoHolePunchRendering(); + } +#endif + auto& quirksManager = GStreamerQuirksManager::singleton(); return quirksManager.supportsVideoHolePunchRendering(); } diff --git a/Source/WebCore/platform/mediastream/CaptureDevice.h b/Source/WebCore/platform/mediastream/CaptureDevice.h index 901bc35f1853c..5e1fc32282e55 100644 --- a/Source/WebCore/platform/mediastream/CaptureDevice.h +++ b/Source/WebCore/platform/mediastream/CaptureDevice.h @@ -32,7 +32,7 @@ namespace WebCore { class CaptureDevice { public: - enum class DeviceType : uint8_t { Unknown, Microphone, Speaker, Camera, Screen, Window, SystemAudio }; + enum class DeviceType : uint8_t { Unknown, Microphone, Speaker, Camera, Screen, Window, SystemAudio, Canvas }; CaptureDevice(const String& persistentId, DeviceType type, const String& label, const String& groupId = emptyString(), bool isEnabled = false, bool isDefault = false, bool isMock = false, bool isEphemeral = false) : m_persistentId(persistentId) diff --git a/Source/WebCore/platform/mediastream/RealtimeMediaSource.cpp b/Source/WebCore/platform/mediastream/RealtimeMediaSource.cpp index f894fef450c68..22f9f493c5fda 100644 --- a/Source/WebCore/platform/mediastream/RealtimeMediaSource.cpp +++ b/Source/WebCore/platform/mediastream/RealtimeMediaSource.cpp @@ -86,6 +86,7 @@ static RealtimeMediaSource::Type toSourceType(CaptureDevice::DeviceType type) case CaptureDevice::DeviceType::SystemAudio: return RealtimeMediaSource::Type::Audio; case CaptureDevice::DeviceType::Camera: + case CaptureDevice::DeviceType::Canvas: case CaptureDevice::DeviceType::Screen: case CaptureDevice::DeviceType::Window: return RealtimeMediaSource::Type::Video; diff --git a/Source/WebCore/platform/mediastream/cocoa/DisplayCaptureSourceCocoa.cpp b/Source/WebCore/platform/mediastream/cocoa/DisplayCaptureSourceCocoa.cpp index 6daea5130e975..b285927d3a364 100644 --- a/Source/WebCore/platform/mediastream/cocoa/DisplayCaptureSourceCocoa.cpp +++ b/Source/WebCore/platform/mediastream/cocoa/DisplayCaptureSourceCocoa.cpp @@ -85,6 +85,7 @@ CaptureSourceOrError DisplayCaptureSourceCocoa::create(const CaptureDevice& devi case CaptureDevice::DeviceType::Microphone: case CaptureDevice::DeviceType::Speaker: case CaptureDevice::DeviceType::Camera: + case CaptureDevice::DeviceType::Canvas: case CaptureDevice::DeviceType::Unknown: ASSERT_NOT_REACHED(); break; diff --git a/Source/WebCore/platform/mediastream/gstreamer/GStreamerMockDevice.cpp b/Source/WebCore/platform/mediastream/gstreamer/GStreamerMockDevice.cpp index 7d84047d125d8..4359329d496db 100644 --- a/Source/WebCore/platform/mediastream/gstreamer/GStreamerMockDevice.cpp +++ b/Source/WebCore/platform/mediastream/gstreamer/GStreamerMockDevice.cpp @@ -59,6 +59,7 @@ GstDevice* webkitMockDeviceCreate(const CaptureDevice& captureDevice) switch (captureDevice.type()) { case CaptureDevice::DeviceType::Camera: + case CaptureDevice::DeviceType::Canvas: case CaptureDevice::DeviceType::Screen: case CaptureDevice::DeviceType::Window: deviceClass = "Video/Source"; diff --git a/Source/WebCore/platform/mediastream/mac/DisplayCaptureManagerCocoa.cpp b/Source/WebCore/platform/mediastream/mac/DisplayCaptureManagerCocoa.cpp index 7e8dd33fefd1b..bb8e63ebea4bd 100644 --- a/Source/WebCore/platform/mediastream/mac/DisplayCaptureManagerCocoa.cpp +++ b/Source/WebCore/platform/mediastream/mac/DisplayCaptureManagerCocoa.cpp @@ -92,6 +92,7 @@ std::optional DisplayCaptureManagerCocoa::captureDeviceWithPersis case CaptureDevice::DeviceType::SystemAudio: case CaptureDevice::DeviceType::Camera: + case CaptureDevice::DeviceType::Canvas: case CaptureDevice::DeviceType::Microphone: case CaptureDevice::DeviceType::Speaker: case CaptureDevice::DeviceType::Unknown: diff --git a/Source/WebCore/platform/mock/MockRealtimeMediaSourceCenter.cpp b/Source/WebCore/platform/mock/MockRealtimeMediaSourceCenter.cpp index 33580b7574eba..355f8f4f63ed2 100644 --- a/Source/WebCore/platform/mock/MockRealtimeMediaSourceCenter.cpp +++ b/Source/WebCore/platform/mock/MockRealtimeMediaSourceCenter.cpp @@ -249,6 +249,7 @@ class MockRealtimeDisplaySourceFactory : public DisplayCaptureFactory { case CaptureDevice::DeviceType::Microphone: case CaptureDevice::DeviceType::Speaker: case CaptureDevice::DeviceType::Camera: + case CaptureDevice::DeviceType::Canvas: case CaptureDevice::DeviceType::SystemAudio: case CaptureDevice::DeviceType::Unknown: ASSERT_NOT_REACHED(); diff --git a/Source/WebKit/GPUProcess/GPUConnectionToWebProcess.cpp b/Source/WebKit/GPUProcess/GPUConnectionToWebProcess.cpp index af9fcd03aa725..4c70f9eafcb8f 100644 --- a/Source/WebKit/GPUProcess/GPUConnectionToWebProcess.cpp +++ b/Source/WebKit/GPUProcess/GPUConnectionToWebProcess.cpp @@ -203,6 +203,7 @@ class GPUProxyForCapture final : public UserMediaCaptureManagerProxy::Connection case CaptureDevice::DeviceType::Microphone: return m_process.get()->allowsAudioCapture(); case CaptureDevice::DeviceType::Camera: + case CaptureDevice::DeviceType::Canvas: if (!m_process.get()->allowsVideoCapture()) return false; #if PLATFORM(IOS_FAMILY) diff --git a/Source/WebKit/UIProcess/Cocoa/UserMediaCaptureManagerProxy.cpp b/Source/WebKit/UIProcess/Cocoa/UserMediaCaptureManagerProxy.cpp index 0278e51700df9..dceae457f3316 100644 --- a/Source/WebKit/UIProcess/Cocoa/UserMediaCaptureManagerProxy.cpp +++ b/Source/WebKit/UIProcess/Cocoa/UserMediaCaptureManagerProxy.cpp @@ -526,6 +526,7 @@ void UserMediaCaptureManagerProxy::createMediaSourceForCaptureDeviceWithConstrai sourceOrError = createMicrophoneSource(device, WTFMove(hashSalts), constraints, pageIdentifier); break; case WebCore::CaptureDevice::DeviceType::Camera: + case WebCore::CaptureDevice::DeviceType::Canvas: sourceOrError = createCameraSource(device, WTFMove(hashSalts), pageIdentifier); break; case WebCore::CaptureDevice::DeviceType::Screen: